BOW Assignment2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "487\n",
      "[('Throughout', 'throughout'), ('Pakistan', 'Pakistan'), (',', ','), ('family', 'family'), ('organization', 'organization'), ('is', 'be'), ('typically', 'typically'), ('patriarchal', 'patriarchal'), (',', ','), ('with', 'with'), ('most', 'most'), ('people', 'people'), ('living', 'live'), ('in', 'in'), ('large', 'large'), ('extended', 'extended'), ('families', 'family'), (',', ','), ('often', 'often'), ('within', 'within'), ('the', 'the'), ('same', 'same'), ('household', 'household'), ('or', 'or'), ('family', 'family'), ('compound', 'compound'), ('.', '.'), ('The', 'the'), ('eldest', 'eld'), ('male', 'male'), (',', ','), ('whether', 'whether'), ('father', 'father'), (',', ','), ('grandfather', 'grandfather'), (',', ','), ('or', 'or'), ('paternal', 'paternal'), ('uncle', 'uncle'), (',', ','), ('usually', 'usually'), ('holds', 'hold'), ('the', 'the'), ('position', 'position'), ('of', 'of'), ('family', 'family'), ('leader', 'leader'), (',', ','), ('making', 'make'), ('significant', 'significant'), ('decisions', 'decision'), ('for', 'for'), ('the', 'the'), ('family', 'family'), ('and', 'and'), ('its', 'its'), ('members', 'member'), ('.', '.'), ('Traditionally', 'traditionally'), (',', ','), ('women', 'woman'), (\"'s\", \"'s\"), ('roles', 'role'), ('have', 'have'), ('been', 'be'), ('secondary', 'secondary'), (',', ','), ('focused', 'focus'), ('on', 'on'), ('domestic', 'domestic'), ('chores', 'chore'), (',', ','), ('and', 'and'), ('fulfilling', 'fulfil'), ('the', 'the'), ('duties', 'duty'), ('of', 'of'), ('a', 'a'), ('wife', 'wife'), ('and', 'and'), ('mother', 'mother'), ('.', '.'), ('However', 'however'), (',', ','), ('in', 'in'), ('Punjab', 'Punjab'), (',', ','), ('cotton', 'cotton'), ('picking', 'picking'), ('is', 'be'), ('exclusively', 'exclusively'), ('done', 'do'), ('by', 'by'), ('women', 'woman'), (',', ','), ('who', 'who'), ('may', 'may'), ('keep', 'keep'), ('the', 'the'), ('earnings', 'earning'), ('for', 'for'), ('themselves', 'themselves'), ('.', '.'), ('\\n\\n', '\\n\\n'), ('In', 'in'), ('affluent', 'affluent'), ('peasant', 'peasant'), ('and', 'and'), ('landowner', 'landowner'), ('households', 'household'), (',', ','), ('as', 'as'), ('well', 'well'), ('as', 'as'), ('urban', 'urban'), ('middle', 'middle'), ('-', '-'), ('class', 'class'), ('families', 'family'), (',', ','), ('the', 'the'), ('practice', 'practice'), ('of', 'of'), ('purdue', 'purdue'), ('(', '('), ('seclusion', 'seclusion'), ('of', 'of'), ('women', 'woman'), (')', ')'), ('is', 'be'), ('still', 'still'), ('common', 'common'), ('.', '.'), ('Women', 'woman'), ('covering', 'cover'), ('their', 'their'), ('heads', 'head'), ('when', 'when'), ('leaving', 'leave'), ('the', 'the'), ('house', 'house'), ('is', 'be'), ('customary', 'customary'), ('in', 'in'), ('these', 'these'), ('communities', 'community'), ('.', '.'), ('Among', 'among'), ('the', 'the'), ('rural', 'rural'), ('poor', 'poor'), (',', ','), ('women', 'woman'), ('have', 'have'), ('responsibilities', 'responsibility'), ('both', 'both'), ('on', 'on'), ('the', 'the'), ('farm', 'farm'), ('and', 'and'), ('at', 'at'), ('home', 'home'), (',', ','), ('without', 'without'), ('observing', 'observe'), ('purdue', 'purdue'), ('.', '.'), ('Homes', 'home'), ('practicing', 'practice'), ('purdue', 'purdue'), ('often', 'often'), ('have', 'have'), ('separate', 'separate'), ('sections', 'section'), ('for', 'for'), ('men', 'man'), ('(', '('), ('markānah', 'markānah'), (')', ')'), ('and', 'and'), ('women', 'woman'), ('(', '('), ('zanānah', 'zanānah'), (')', ')'), ('to', 'to'), ('maintain', 'maintain'), ('privacy', 'privacy'), ('.', '.'), ('\\n\\n', '\\n\\n'), ('The', 'the'), ('subordinate', 'subordinate'), ('status', 'status'), ('of', 'of'), ('women', 'woman'), ('in', 'in'), ('Pakistan', 'Pakistan'), ('is', 'be'), ('also', 'also'), ('evident', 'evident'), ('in', 'in'), ('practices', 'practice'), ('such', 'such'), ('as', 'as'), ('\"', '\"'), ('honour', 'honour'), ('killings', 'killing'), (',', ','), ('\"', '\"'), ('where', 'where'), ('a', 'a'), ('woman', 'woman'), ('may', 'may'), ('be', 'be'), ('killed', 'kill'), ('by', 'by'), ('a', 'a'), ('male', 'male'), ('relative', 'relative'), ('for', 'for'), ('allegedly', 'allegedly'), ('bringing', 'bring'), ('dishonour', 'dishonour'), ('to', 'to'), ('the', 'the'), ('family', 'family'), ('or', 'or'), ('clan', 'clan'), ('.', '.'), ('\\n\\n', '\\n\\n'), ('While', 'while'), ('Western', 'western'), ('education', 'education'), ('and', 'and'), ('lifestyles', 'lifestyle'), ('have', 'have'), ('diminished', 'diminish'), ('the', 'the'), ('practice', 'practice'), ('of', 'of'), ('purdue', 'purdue'), ('among', 'among'), ('the', 'the'), ('wealthiest', 'wealthy'), ('Pakistani', 'Pakistani'), (',', ','), ('attitudes', 'attitude'), ('toward', 'toward'), ('women', 'woman'), ('in', 'in'), ('society', 'society'), ('and', 'and'), ('the', 'the'), ('family', 'family'), ('are', 'be'), ('often', 'often'), ('perceived', 'perceive'), ('as', 'as'), ('outdated', 'outdate'), ('.', '.'), ('However', 'however'), (',', ','), ('change', 'change'), ('is', 'be'), ('happening', 'happen'), ('more', 'more'), ('rapidly', 'rapidly'), ('among', 'among'), ('the', 'the'), ('urban', 'urban'), ('middle', 'middle'), ('class', 'class'), (',', ','), ('influenced', 'influence'), ('by', 'by'), ('increased', 'increase'), ('exposure', 'exposure'), ('to', 'to'), ('Western', 'western'), ('values', 'value'), (',', ','), ('women', 'woman'), ('entering', 'enter'), ('the', 'the'), ('workforce', 'workforce'), ('and', 'and'), ('government', 'government'), ('service', 'service'), (',', ','), ('and', 'and'), ('encouragement', 'encouragement'), ('of', 'of'), ('women', 'woman'), (\"'s\", \"'s\"), ('education', 'education'), ('.', '.'), ('Some', 'some'), ('women', 'woman'), ('have', 'have'), ('achieved', 'achieve'), ('prominence', 'prominence'), ('in', 'in'), ('various', 'various'), ('professions', 'profession'), ('and', 'and'), ('political', 'political'), ('roles', 'role'), ('.', '.'), ('\\n\\n', '\\n\\n'), ('In', 'in'), ('traditional', 'traditional'), ('Pakistani', 'pakistani'), ('social', 'social'), ('organization', 'organization'), (',', ','), ('kinship', 'kinship'), ('is', 'be'), ('central', 'central'), (',', ','), ('unlike', 'unlike'), ('the', 'the'), ('caste', 'caste'), ('system', 'system'), ('in', 'in'), ('India', 'India'), ('.', '.'), ('The', 'the'), ('baradari', 'baradari'), ('(', '('), ('patrilineage', 'patrilineage'), (')', ')'), ('is', 'be'), ('a', 'a'), ('crucial', 'crucial'), ('social', 'social'), ('institution', 'institution'), (',', ','), ('with', 'with'), ('endgame', 'endgame'), ('commonly', 'commonly'), ('practiced', 'practice'), ('.', '.'), ('Marriage', 'marriage'), ('within', 'within'), ('the', 'the'), ('baradari', 'baradari'), ('or', 'or'), ('kinship', 'kinship'), ('group', 'group'), ('is', 'be'), ('preferred', 'prefer'), ('.', '.'), ('Lineage', 'lineage'), ('elders', 'elder'), ('form', 'form'), ('councils', 'council'), ('to', 'to'), ('settle', 'settle'), ('disputes', 'dispute'), ('and', 'and'), ('represent', 'represent'), ('the', 'the'), ('lineage', 'lineage'), ('externally', 'externally'), ('.', '.'), ('\\n\\n', '\\n\\n'), ('Pakistani', 'pakistani'), ('clothing', 'clothing'), (',', ','), ('similar', 'similar'), ('to', 'to'), ('Indian', 'indian'), ('styles', 'style'), (',', ','), ('includes', 'include'), ('the', 'the'), ('scalar', 'scalar'), ('-', '-'), ('kami', 'kami'), ('combination', 'combination'), ('for', 'for'), ('both', 'both'), ('men', 'man'), ('and', 'and'), ('women', 'woman'), ('.', '.'), ('Among', 'among'), ('conservative', 'conservative'), ('Muslim', 'muslim'), ('communities', 'community'), (',', ','), ('women', 'woman'), ('may', 'may'), ('wear', 'wear'), ('the', 'the'), ('burma', 'burma'), ('.', '.'), ('Western', 'western'), ('attire', 'attire'), ('is', 'be'), ('popular', 'popular'), ('among', 'among'), ('urban', 'urban'), ('youth', 'youth'), (',', ','), ('often', 'often'), ('blending', 'blend'), ('with', 'with'), ('Pakistani', 'pakistani'), ('styles', 'style'), ('.', '.'), ('\\n\\n', '\\n\\n'), ('Pakistani', 'pakistani'), ('cuisine', 'cuisine'), ('shares', 'share'), ('similarities', 'similarity'), ('with', 'with'), ('Indian', 'indian'), ('cuisine', 'cuisine'), (',', ','), ('featuring', 'feature'), ('curry', 'curry'), ('dishes', 'dish'), (',', ','), ('various', 'various'), ('vegetables', 'vegetable'), (',', ','), ('spices', 'spice'), (',', ','), ('yogurt', 'yogurt'), (',', ','), ('and', 'and'), ('meats', 'meat'), ('like', 'like'), ('chicken', 'chicken'), (',', ','), ('mutton', 'mutton'), (',', ','), ('and', 'and'), ('lamb', 'lamb'), ('.', '.'), ('Rentals', 'rental'), ('and', 'and'), ('wheat', 'wheat'), ('bread', 'bread'), ('are', 'be'), ('staples', 'staple'), ('.', '.'), ('Hot', 'hot'), ('tea', 'tea'), (',', ','), ('last', 'last'), (',', ','), ('sheet', 'sheet'), (',', ','), ('and', 'and'), ('lemonade', 'lemonade'), ('are', 'be'), ('common', 'common'), ('beverages', 'beverage'), ('.', '.'), ('Alcoholic', 'alcoholic'), ('beverages', 'beverage'), ('are', 'be'), ('culturally', 'culturally'), ('inappropriate', 'inappropriate'), (',', ','), ('although', 'although'), ('there', 'there'), ('are', 'be'), ('domestic', 'domestic'), ('breweries', 'brewery'), ('and', 'and'), ('distilleries', 'distillery'), ('.', '.'), ('\\n\\n', '\\n\\n'), ('Muslim', 'Muslim'), ('Pakistani', 'Pakistani'), ('celebrate', 'celebrate'), ('Islamic', 'islamic'), ('holidays', 'holiday'), ('such', 'such'), ('as', 'as'), ('Eid', 'Eid'), ('al', 'al'), ('-', '-'), ('Fit', 'Fit'), (',', ',')]\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import spacy\n",
    "nlp=spacy.load(\"en_core_web_sm\")\n",
    "from autocorrect import Speller\n",
    "spell = Speller()\n",
    "import nltk\n",
    "from nltk import sent_tokenize\n",
    "path = 'D:\\\\NLP\\\\'\n",
    "filepath = os.listdir(path)\n",
    "\n",
    "D_S = []\n",
    "\n",
    "def filetext(filename):\n",
    "    for file in filename:\n",
    "        text = open(\"D:\\\\NLP\\\\\" + file, \"r\", encoding=\"utf-8\").read()\n",
    "        index = [text, type(text)]\n",
    "        D_S.append(index)\n",
    "\n",
    "filetext(filepath)\n",
    "\n",
    "\n",
    "\n",
    "def spell_correction(text):\n",
    "    text=spell(text)\n",
    "    return text\n",
    "\n",
    "def tokenize(text):\n",
    "    tokens = nlp(text)\n",
    "    return tokens\n",
    "\n",
    "def StopWords_Removal(tokens):\n",
    "    stopword_removal = [t.text for t in tokens if not  t.is_stop]\n",
    "    return stopword_removal\n",
    "\n",
    "def POS_Tagging(stopword_removal):\n",
    "    pos_tags = [(t.text, t.pos_, t.tag_) for t in stopword_removal]\n",
    "    return pos_tags\n",
    "\n",
    "\n",
    "def convert_to_lowercase(pos_tags):\n",
    "    tokens_in_lowercase = [w.lower() for w in pos_tags]\n",
    "    return tokens_in_lowercase\n",
    "\n",
    "def Lemmatization(tokens_in_lowercase):\n",
    "    lemmas = [(token.text, token.lemma_) for token in tokens_in_lowercase]\n",
    "    return lemmas\n",
    "\n",
    "DSP=[]\n",
    "for ds in D_S:\n",
    "    text = ds[0]\n",
    "    DSP_item = []\n",
    "    corrected_text = spell_correction(text)\n",
    "    DSP_item.append(corrected_text) \n",
    "    \n",
    "    tokkens = tokenize(corrected_text)\n",
    "    DSP_item.append(tokkens)\n",
    "\n",
    "    DSP_item.append(POS_Tagging(tokkens))\n",
    "\n",
    "    T_WSW = StopWords_Removal(tokkens)\n",
    "    DSP_item.append(T_WSW)                                                                                                                                 \n",
    "\n",
    "    T_LLC = convert_to_lowercase(T_WSW)\n",
    "    DSP_item.append(T_LLC) \n",
    "    lemmas = Lemmatization(tokkens)\n",
    "    DSP_item.append(lemmas) \n",
    "    DSP.append(DSP_item)\n",
    "\n",
    "BOW=[]\n",
    "for ds in DSP:\n",
    "    lemmas = ds[-1]\n",
    "    for l in lemmas:\n",
    "        BOW.append(l)\n",
    "print(len(BOW))\n",
    "\n",
    "print(BOW)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}