SPENSER_notebook_12.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import csv\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open the file with the unemployed people\n",
    "df_unemployed_dir = r'C:\\Users\\b9055315\\Documents\\PhD_PROJECT\\Synthetic_population_developement\\SPENSER\\Data\\2019\\economic_activity' # use your path\n",
    "df_unemployed_file = os.path.join(df_unemployed_dir, \"df_persons_NE_unemployed_2021112.csv\")\n",
    "df_unemployed = pd.read_csv(df_unemployed_file, index_col=None, header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "448818"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Get only those that are <16 years old only\",\n",
    "df_unemployed_under16 = df_unemployed.loc[(df_unemployed['Age']<16)]\n",
    "\n",
    "len(df_unemployed_under16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\ArcGIS\\Pro\\bin\\Python\\envs\\arcgispro-py3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "# Update their economic_activity to 'Inactive'\n",
    "df_unemployed_under16['Economic_activity'] = 'Inactive'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\ArcGIS\\Pro\\bin\\Python\\envs\\arcgispro-py3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "# Update their occupation to 'Child student'\n",
    "df_unemployed_under16['Occupation'] = 'Child student'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\ArcGIS\\Pro\\bin\\Python\\envs\\arcgispro-py3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "# Update their income to 0\n",
    "df_unemployed_under16['Income'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_persons_NE_children_20220309 = df_unemployed_under16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "448818"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_persons_NE_children_20220309)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open the file with the persons with occupation and income (output from spenser_notebook_8)\n",
    "df_occupation_income_dir = r'C:\\Users\\b9055315\\Documents\\PhD_PROJECT\\Synthetic_population_developement\\SPENSER\\Data\\2019\\income' # use your path\n",
    "df_occupation_income_file = os.path.join(df_occupation_income_dir, \"df_income_new_2_20222307_correct.csv\")\n",
    "df_occupation_income = pd.read_csv(df_occupation_income_file, index_col=None, header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open the file with the inactive persons and income (output from spenser_notebook_11)\n",
    "df_income_inactive_dir = r'C:\\Users\\b9055315\\Documents\\PhD_PROJECT\\Synthetic_population_developement\\SPENSER\\Data\\2019\\income\\inactive' # use your path\n",
    "df_income_inactive_file = os.path.join(df_income_inactive_dir, \"df_persons_NE_inactive_5categories_income_export_20220306_correct.csv\")\n",
    "df_income_inactive = pd.read_csv(df_income_inactive_file, index_col=None, header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2645517"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# concatenate the last three dataframes and check if total number is the total population of the area of study:\n",
    "df_persons_NE_sociodemographics = pd.concat([df_occupation_income, df_income_inactive, df_persons_NE_children_20220309])\n",
    "\n",
    "len(df_persons_NE_sociodemographics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the data:\n",
    "df_persons_NE_sociodemographics.to_csv(r'C:\\Users\\b9055315\\Documents\\PhD_PROJECT\\Synthetic_population_developement\\SPENSER\\Data\\2019\\Attributes_merged\\df_persons_NE_sociodemographics_20220309.csv', encoding='utf-8', header=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Analysis of attribute values. 3 groups of people: age 0-15, 16-64 and 65+"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_persons_NE_sociodemographics_0_15 = df_persons_NE_sociodemographics.loc[(df_persons_NE_sociodemographics['Age'] < 16)]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_0_15['Children_dependency'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Single'], dtype=object)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_0_15['Marital_status'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Inactive'], dtype=object)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_0_15['Economic_activity'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Child student'], dtype=object)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_0_15['Occupation'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.])"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_0_15['Income'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_persons_NE_sociodemographics_16_64 = df_persons_NE_sociodemographics.loc[(df_persons_NE_sociodemographics['Age'] >= 16) & (df_persons_NE_sociodemographics['Age'] <= 64)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Married', 'Couple', 'Single'], dtype=object)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_16_64['Marital_status'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False,  True])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_16_64['Children_dependency'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Employed', 'Unemployed', 'Inactive'], dtype=object)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_16_64['Economic_activity'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 'Retired',\n",
       "       'Long term sick', 'Looking after home family', 'Other', 'Student'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_16_64['Occupation'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1040.0"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_16_64['Income'].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99870.0"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_16_64['Income'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_persons_NE_sociodemographics_65_120 = df_persons_NE_sociodemographics.loc[(df_persons_NE_sociodemographics['Age'] >= 65) & (df_persons_NE_sociodemographics['Age'] <= 120)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Married', 'Single', 'Couple'], dtype=object)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_65_120['Marital_status'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True, False])"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_65_120['Children_dependency'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Employed', 'Unemployed', 'Inactive'], dtype=object)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_65_120['Economic_activity'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 'Retired'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_65_120['Occupation'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1490.0"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_65_120['Income'].min()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99860.0"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_65_120['Income'].max()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Adult_Similar_age                 bool\n",
       "Age                            float64\n",
       "Area_MSOA_x                     object\n",
       "Area_OA_x                       object\n",
       "Children_dependency               bool\n",
       "Economic_activity               object\n",
       "Ethnic                         float64\n",
       "HID_AreaOA_x                    object\n",
       "HID_x                          float64\n",
       "HRPID_AreaMSOA_x                object\n",
       "Income                         float64\n",
       "LC4202_C_CARSNO_x              float64\n",
       "LC4202_C_ETHHUK11_x            float64\n",
       "LC4404_C_SIZHUK11_x            float64\n",
       "LC4408_C_AHTHUK11_x            float64\n",
       "LC4605_C_NSSEC_x               float64\n",
       "Marital_status                  object\n",
       "NSSEC                          float64\n",
       "Occupation                      object\n",
       "PID                            float64\n",
       "PID_AreaMSOA                    object\n",
       "QS420_CELL                     float64\n",
       "Same_ethnic                       bool\n",
       "Sex                            float64\n",
       "Total_Children_in_household    float64\n",
       "Total_People_in_household      float64\n",
       "Unnamed: 0                       int64\n",
       "Unnamed: 0.1                     int64\n",
       "Unnamed: 0.1.1                 float64\n",
       "Unnamed: 0.1.1.1               float64\n",
       "Unnamed: 0.1.1.1.1             float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Select the columns to be kept:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_list= ['PID', 'PID_AreaMSOA', 'HID_x', 'HID_AreaOA_x', 'Area_MSOA_x', 'Area_OA_x', 'HRPID_AreaMSOA_x',\n",
    "              'Sex', 'Age', 'Ethnic', 'Marital_status', 'Children_dependency', 'Adult_Similar_age',\n",
    "              'Total_People_in_household', 'Total_Children_in_household', 'Same_ethnic',\n",
    "              'Economic_activity', 'Occupation', 'Income',\n",
    "              'LC4202_C_CARSNO_x', 'LC4202_C_ETHHUK11_x', 'LC4404_C_SIZHUK11_x', 'LC4408_C_AHTHUK11_x',\n",
    "              'QS420_CELL']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new dataframe with only the selected columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_persons_NE_sociodemographics_clean = df_persons_NE_sociodemographics[['PID', 'PID_AreaMSOA', 'HID_x', 'HID_AreaOA_x', 'Area_MSOA_x', 'Area_OA_x', 'HRPID_AreaMSOA_x',\n",
    "              'Sex', 'Age', 'Ethnic', 'Marital_status', 'Children_dependency', 'Adult_Similar_age',\n",
    "              'Total_People_in_household', 'Total_Children_in_household', 'Same_ethnic',\n",
    "              'Economic_activity', 'Occupation', 'Income',\n",
    "              'LC4202_C_CARSNO_x', 'LC4202_C_ETHHUK11_x', 'LC4404_C_SIZHUK11_x', 'LC4408_C_AHTHUK11_x',\n",
    "              'QS420_CELL']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2645517"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_persons_NE_sociodemographics_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PID                            float64\n",
       "PID_AreaMSOA                    object\n",
       "HID_x                          float64\n",
       "HID_AreaOA_x                    object\n",
       "Area_MSOA_x                     object\n",
       "Area_OA_x                       object\n",
       "HRPID_AreaMSOA_x                object\n",
       "Sex                            float64\n",
       "Age                            float64\n",
       "Ethnic                         float64\n",
       "Marital_status                  object\n",
       "Children_dependency               bool\n",
       "Adult_Similar_age                 bool\n",
       "Total_People_in_household      float64\n",
       "Total_Children_in_household    float64\n",
       "Same_ethnic                       bool\n",
       "Economic_activity               object\n",
       "Occupation                      object\n",
       "Income                         float64\n",
       "LC4202_C_CARSNO_x              float64\n",
       "LC4202_C_ETHHUK11_x            float64\n",
       "LC4404_C_SIZHUK11_x            float64\n",
       "LC4408_C_AHTHUK11_x            float64\n",
       "QS420_CELL                     float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_clean.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Read the mobility attributes (driving licence, car acess and bike access)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\ArcGIS\\Pro\\bin\\Python\\envs\\arcgispro-py3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2785: DtypeWarning: Columns (16) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "# Bike access:\n",
    "df_persons_NE_bike_dir = r'C:\\Users\\b9055315\\Documents\\PhD_PROJECT\\Synthetic_population_developement\\SPENSER\\Data\\2019\\Bike_access' # use your path\n",
    "df_persons_NE_bike_file = os.path.join(df_persons_NE_bike_dir, \"df_persons_NE_bike_updated.csv\")\n",
    "df_persons_NE_bike = pd.read_csv(df_persons_NE_bike_file, index_col=None, header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2645517"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_persons_NE_bike)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0               int64\n",
       "Unnamed: 0.1             int64\n",
       "PID_AreaMSOA            object\n",
       "PID                      int64\n",
       "Area_MSOA_x             object\n",
       "Area_OA_x               object\n",
       "Sex                      int64\n",
       "Age                      int64\n",
       "Ethnic                   int64\n",
       "HID_x                    int64\n",
       "HID_AreaOA_x            object\n",
       "LC4408_C_AHTHUK11_x      int64\n",
       "LC4404_C_SIZHUK11_x      int64\n",
       "LC4605_C_NSSEC_x         int64\n",
       "LC4202_C_CARSNO_x        int64\n",
       "LC4202_C_ETHHUK11_x      int64\n",
       "HRPID_AreaMSOA          object\n",
       "LC4605_C_NSSEC_y       float64\n",
       "QS420_CELL_x             int64\n",
       "Bike_access               bool\n",
       "dtype: object"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_bike.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Car access (driving licence is included in this file)\n",
    "df_persons_NE_car_access_dir = r'C:\\Users\\b9055315\\Documents\\PhD_PROJECT\\Synthetic_population_developement\\SPENSER\\Data\\2019\\Driving_licence' # use your path\n",
    "df_persons_NE_car_access_file = os.path.join(df_persons_NE_car_access_dir, \"df_persons_NE_car_access_20222306_correct.csv\")\n",
    "df_persons_NE_car_access = pd.read_csv(df_persons_NE_car_access_file, index_col=None, header=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2645517"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_persons_NE_car_access)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0               int64\n",
       "Unnamed: 0.1             int64\n",
       "Unnamed: 0.1.1           int64\n",
       "PID_AreaMSOA            object\n",
       "PID                      int64\n",
       "Area_MSOA_x             object\n",
       "Area_OA_x               object\n",
       "Sex                      int64\n",
       "Age                      int64\n",
       "Ethnic                   int64\n",
       "HID_x                    int64\n",
       "HID_AreaOA_x            object\n",
       "LC4408_C_AHTHUK11_x      int64\n",
       "LC4404_C_SIZHUK11_x      int64\n",
       "LC4605_C_NSSEC_x         int64\n",
       "LC4202_C_CARSNO_x        int64\n",
       "LC4202_C_ETHHUK11_x      int64\n",
       "HRPID_AreaMSOA          object\n",
       "LC4605_C_NSSEC_y       float64\n",
       "QS420_CELL_x             int64\n",
       "RUC11                   object\n",
       "Driving_license           bool\n",
       "Car_access                bool\n",
       "dtype: object"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_car_access.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Join df_households_NE with df_MSOA_OA_2011_NE_list in order to get MSOA level within \n",
    "#df_households_NE and then pass the Area_OA to df_persons_NE based on the MSOA level value\n",
    "df_persons_NE_mobility = df_persons_NE_car_access.merge(df_persons_NE_bike, left_on='PID_AreaMSOA', right_on='PID_AreaMSOA', how='left')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2645517"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_persons_NE_mobility)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0_x               int64\n",
       "Unnamed: 0.1_x             int64\n",
       "Unnamed: 0.1.1             int64\n",
       "PID_AreaMSOA              object\n",
       "PID_x                      int64\n",
       "Area_MSOA_x_x             object\n",
       "Area_OA_x_x               object\n",
       "Sex_x                      int64\n",
       "Age_x                      int64\n",
       "Ethnic_x                   int64\n",
       "HID_x_x                    int64\n",
       "HID_AreaOA_x_x            object\n",
       "LC4408_C_AHTHUK11_x_x      int64\n",
       "LC4404_C_SIZHUK11_x_x      int64\n",
       "LC4605_C_NSSEC_x_x         int64\n",
       "LC4202_C_CARSNO_x_x        int64\n",
       "LC4202_C_ETHHUK11_x_x      int64\n",
       "HRPID_AreaMSOA_x          object\n",
       "LC4605_C_NSSEC_y_x       float64\n",
       "QS420_CELL_x_x             int64\n",
       "RUC11                     object\n",
       "Driving_license             bool\n",
       "Car_access                  bool\n",
       "Unnamed: 0_y               int64\n",
       "Unnamed: 0.1_y             int64\n",
       "PID_y                      int64\n",
       "Area_MSOA_x_y             object\n",
       "Area_OA_x_y               object\n",
       "Sex_y                      int64\n",
       "Age_y                      int64\n",
       "Ethnic_y                   int64\n",
       "HID_x_y                    int64\n",
       "HID_AreaOA_x_y            object\n",
       "LC4408_C_AHTHUK11_x_y      int64\n",
       "LC4404_C_SIZHUK11_x_y      int64\n",
       "LC4605_C_NSSEC_x_y         int64\n",
       "LC4202_C_CARSNO_x_y        int64\n",
       "LC4202_C_ETHHUK11_x_y      int64\n",
       "HRPID_AreaMSOA_y          object\n",
       "LC4605_C_NSSEC_y_y       float64\n",
       "QS420_CELL_x_y             int64\n",
       "Bike_access                 bool\n",
       "dtype: object"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_mobility.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_persons_NE_mobility_clean = df_persons_NE_mobility[['PID_AreaMSOA', 'Driving_license', 'Car_access', 'Bike_access','Age_x']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PID_AreaMSOA       object\n",
       "Driving_license      bool\n",
       "Car_access           bool\n",
       "Bike_access          bool\n",
       "Age_x               int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_mobility_clean.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_persons_NE_mobility_clean_0_15 = df_persons_NE_mobility_clean.loc[(df_persons_NE_mobility_clean['Age_x'] < 16)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_mobility_clean_0_15['Driving_license'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_mobility_clean_0_15['Car_access'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False,  True])"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_mobility_clean_0_15['Bike_access'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_persons_NE_mobility_clean_16_64 = df_persons_NE_mobility_clean.loc[(df_persons_NE_mobility_clean['Age_x'] >= 16) & (df_persons_NE_mobility_clean['Age_x'] <= 64)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True, False])"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_mobility_clean_16_64['Driving_license'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True, False])"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_mobility_clean_16_64['Car_access'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True, False])"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_mobility_clean_16_64['Bike_access'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_persons_NE_mobility_clean_65plus = df_persons_NE_mobility_clean.loc[(df_persons_NE_mobility_clean['Age_x'] >= 65) & (df_persons_NE_mobility_clean['Age_x'] <= 120)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True, False])"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_mobility_clean_65plus['Driving_license'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True, False])"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_mobility_clean_65plus['Car_access'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False,  True])"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_mobility_clean_65plus['Bike_access'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save dataframe\n",
    "df_persons_NE_mobility_20220309 = df_persons_NE_mobility_clean\n",
    "df_persons_NE_mobility_20220309.to_csv(r'C:\\Users\\b9055315\\Documents\\PhD_PROJECT\\Synthetic_population_developement\\SPENSER\\Data\\2019\\Attributes_merged\\df_persons_NE_mobility_20220309.csv', encoding='utf-8', header=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2645517"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_persons_NE_mobility_20220309)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Join df_households_NE with df_MSOA_OA_2011_NE_list in order to get MSOA level within \n",
    "#df_households_NE and then pass the Area_OA to df_persons_NE based on the MSOA level value\n",
    "df_persons_NE_sociodemographics_mobility = df_persons_NE_sociodemographics_clean.merge(df_persons_NE_mobility_clean, left_on='PID_AreaMSOA', right_on='PID_AreaMSOA', how='left')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2645517"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_persons_NE_sociodemographics_mobility)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2645517"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check if there are duplicates:\n",
    "df_persons_NE_sociodemographics_mobility_1 = df_persons_NE_sociodemographics_mobility\n",
    "\n",
    "# Remove duplicates\n",
    "df_persons_NE_sociodemographics_mobility_1 = df_persons_NE_sociodemographics_mobility_1.drop_duplicates(subset='PID_AreaMSOA',keep = False)\n",
    "\n",
    "\n",
    "len(df_persons_NE_sociodemographics_mobility_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PID                            float64\n",
       "PID_AreaMSOA                    object\n",
       "HID_x                          float64\n",
       "HID_AreaOA_x                    object\n",
       "Area_MSOA_x                     object\n",
       "Area_OA_x                       object\n",
       "HRPID_AreaMSOA_x                object\n",
       "Sex                            float64\n",
       "Age                            float64\n",
       "Ethnic                         float64\n",
       "Marital_status                  object\n",
       "Children_dependency               bool\n",
       "Adult_Similar_age                 bool\n",
       "Total_People_in_household      float64\n",
       "Total_Children_in_household    float64\n",
       "Same_ethnic                       bool\n",
       "Economic_activity               object\n",
       "Occupation                      object\n",
       "Income                         float64\n",
       "LC4202_C_CARSNO_x              float64\n",
       "LC4202_C_ETHHUK11_x            float64\n",
       "LC4404_C_SIZHUK11_x            float64\n",
       "LC4408_C_AHTHUK11_x            float64\n",
       "QS420_CELL                     float64\n",
       "Driving_license                   bool\n",
       "Car_access                        bool\n",
       "Bike_access                       bool\n",
       "Age_x                            int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_mobility.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_persons_NE_sociodemographics_mobility_clean = df_persons_NE_sociodemographics_mobility[['PID', 'PID_AreaMSOA', 'HID_x', 'HID_AreaOA_x', 'Area_MSOA_x', 'Area_OA_x', 'HRPID_AreaMSOA_x',\n",
    "              'Sex', 'Age', 'Ethnic', 'Marital_status', 'Children_dependency', 'Adult_Similar_age',\n",
    "              'Total_People_in_household', 'Total_Children_in_household', 'Same_ethnic',\n",
    "              'Economic_activity', 'Occupation', 'Income', 'Driving_license', 'Car_access', 'Bike_access',\n",
    "              'LC4202_C_CARSNO_x', 'LC4202_C_ETHHUK11_x', 'LC4404_C_SIZHUK11_x', 'LC4408_C_AHTHUK11_x',\n",
    "              'QS420_CELL']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PID                            float64\n",
       "PID_AreaMSOA                    object\n",
       "HID_x                          float64\n",
       "HID_AreaOA_x                    object\n",
       "Area_MSOA_x                     object\n",
       "Area_OA_x                       object\n",
       "HRPID_AreaMSOA_x                object\n",
       "Sex                            float64\n",
       "Age                            float64\n",
       "Ethnic                         float64\n",
       "Marital_status                  object\n",
       "Children_dependency               bool\n",
       "Adult_Similar_age                 bool\n",
       "Total_People_in_household      float64\n",
       "Total_Children_in_household    float64\n",
       "Same_ethnic                       bool\n",
       "Economic_activity               object\n",
       "Occupation                      object\n",
       "Income                         float64\n",
       "Driving_license                   bool\n",
       "Car_access                        bool\n",
       "Bike_access                       bool\n",
       "LC4202_C_CARSNO_x              float64\n",
       "LC4202_C_ETHHUK11_x            float64\n",
       "LC4404_C_SIZHUK11_x            float64\n",
       "LC4408_C_AHTHUK11_x            float64\n",
       "QS420_CELL                     float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_mobility_clean.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2645517"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check if there are duplicates:\n",
    "df_persons_NE_sociodemographics_mobility_clean_1 = df_persons_NE_sociodemographics_mobility_clean\n",
    "\n",
    "# Remove duplicates\n",
    "df_persons_NE_sociodemographics_mobility_clean_1 = df_persons_NE_sociodemographics_mobility_clean_1.drop_duplicates(subset='PID_AreaMSOA',keep = False)\n",
    "\n",
    "\n",
    "len(df_persons_NE_sociodemographics_mobility_clean_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PID                            float64\n",
       "PID_AreaMSOA                    object\n",
       "HID_x                          float64\n",
       "HID_AreaOA_x                    object\n",
       "Area_MSOA_x                     object\n",
       "Area_OA_x                       object\n",
       "HRPID_AreaMSOA_x                object\n",
       "Sex                            float64\n",
       "Age                            float64\n",
       "Ethnic                         float64\n",
       "Marital_status                  object\n",
       "Children_dependency               bool\n",
       "Adult_Similar_age                 bool\n",
       "Total_People_in_household      float64\n",
       "Total_Children_in_household    float64\n",
       "Same_ethnic                       bool\n",
       "Economic_activity               object\n",
       "Occupation                      object\n",
       "Income                         float64\n",
       "Driving_license                   bool\n",
       "Car_access                        bool\n",
       "Bike_access                       bool\n",
       "LC4202_C_CARSNO_x              float64\n",
       "LC4202_C_ETHHUK11_x            float64\n",
       "LC4404_C_SIZHUK11_x            float64\n",
       "LC4408_C_AHTHUK11_x            float64\n",
       "QS420_CELL                     float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_mobility_clean.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Rename the column names of 'df_persons_NE'\n",
    "df_persons_NE_sociodemographics_mobility_clean = df_persons_NE_sociodemographics_mobility_clean.rename({'HID_AreaOA_x': 'HID_AreaOA',\n",
    "                                                                       'HID_x': 'HID', \n",
    "                                                                       'Area_MSOA_x': 'Area_MSOA', \n",
    "                                                                       'Area_OA_x': 'Area_OA',\n",
    "                                                                      'HRPID_AreaMSOA_x': 'HRPID_AreaMSOA',\n",
    "                                                                      'LC4202_C_CARSNO_x': 'LC4202_C_CARSNO',\n",
    "                                                                      'LC4202_C_ETHHUK11_x': 'LC4202_C_ETHHUK11',\n",
    "                                                                      'LC4404_C_SIZHUK11_x': 'LC4404_C_SIZHUK11',\n",
    "                                                                      'LC4408_C_AHTHUK11_x': 'LC4408_C_AHTHUK11'}, axis=1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PID                            float64\n",
       "PID_AreaMSOA                    object\n",
       "HID                            float64\n",
       "HID_AreaOA                      object\n",
       "Area_MSOA                       object\n",
       "Area_OA                         object\n",
       "HRPID_AreaMSOA                  object\n",
       "Sex                            float64\n",
       "Age                            float64\n",
       "Ethnic                         float64\n",
       "Marital_status                  object\n",
       "Children_dependency               bool\n",
       "Adult_Similar_age                 bool\n",
       "Total_People_in_household      float64\n",
       "Total_Children_in_household    float64\n",
       "Same_ethnic                       bool\n",
       "Economic_activity               object\n",
       "Occupation                      object\n",
       "Income                         float64\n",
       "Driving_license                   bool\n",
       "Car_access                        bool\n",
       "Bike_access                       bool\n",
       "LC4202_C_CARSNO                float64\n",
       "LC4202_C_ETHHUK11              float64\n",
       "LC4404_C_SIZHUK11              float64\n",
       "LC4408_C_AHTHUK11              float64\n",
       "QS420_CELL                     float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_persons_NE_sociodemographics_mobility_clean.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save dataframe\n",
    "df_persons_NE_sociodemographics_mobility_clean_export_20220309_correct = df_persons_NE_sociodemographics_mobility_clean\n",
    "df_persons_NE_sociodemographics_mobility_clean_export_20220309_correct.to_csv(r'C:\\Users\\b9055315\\Documents\\PhD_PROJECT\\Synthetic_population_developement\\SPENSER\\Data\\2019\\Attributes_merged\\df_persons_NE_sociodemographics_mobility_clean_export_20220309_correct.csv', encoding='utf-8', header=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2645517"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_persons_NE_sociodemographics_mobility_clean_export_20220309_correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2645517"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check if there are duplicates:\n",
    "df_persons_NE_sociodemographics_mobility_clean_export_20220309_correct_1 = df_persons_NE_sociodemographics_mobility_clean_export_20220309_correct\n",
    "\n",
    "# Remove duplicates\n",
    "df_persons_NE_sociodemographics_mobility_clean_export_20220309_correct_1 = df_persons_NE_sociodemographics_mobility_clean_export_20220309_correct_1.drop_duplicates(subset='PID_AreaMSOA',keep = False)\n",
    "\n",
    "\n",
    "len(df_persons_NE_sociodemographics_mobility_clean_export_20220309_correct_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2645517"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_persons_NE_sociodemographics_mobility_clean_export_20220309_correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}