News-classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./outfile.csv',header=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)\n",
    "df = df.reset_index(drop=True)\n",
    "df['Unnamed: 0'].drop(index=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\manish-laptop\\AppData\\Roaming\\Python\\Python37\\site-packages\\dateutil\\parser\\_parser.py:1206: UnknownTimezoneWarning: tzname IST identified but not understood.  Pass `tzinfos` argument in order to correctly return a timezone-aware datetime.  In a future version, this will raise an exception.\n",
      "  category=UnknownTimezoneWarning)\n"
     ]
    }
   ],
   "source": [
    "df['post_date']= pd.to_datetime(df['post_date'])\n",
    "df['post_date'] = df['post_date'].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>post_title</th>\n",
       "      <th>post_text</th>\n",
       "      <th>post_date</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>15th edition of Golden leaf India awards presented</td>\n",
       "      <td>15th edition golden leaf india awards southern tea competition jointly organized united planters association southern india tea board ahmedabad saw 66 shortlisted teas qualified screening panel jury comprising renowned tasters major buyers arun kumar convenor competition said bombay trading corporation five awards kanan devan hills plantations company pvt ltd kdhp united nilgiri tea estates company ltd shared four awards devon plantations industries ltd joonktollee tea industries ltd harrisons malayalam limitedsbu b kodanad estate shared three awards harrisons malayalam limitedsbu two awards waterfall estate pvt ltd woodbriar group shared one award vigneshwar estate tea factory six awards bought leaf factory segment kdhp managing director k mathew abraham said awards company high range region recognition high quality standards maintained company safe sustainable agronomical practices kdhp largest plantation company south india 41 awards years munnar region facilities located besides indian market exports tea europe us russian asian countries</td>\n",
       "      <td>2019-04-30 23:30:00+05:30</td>\n",
       "      <td>agriculture</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Multi-state initiative in South India to help paddy farmers</td>\n",
       "      <td>kochi group paddy farmers kerala taken initiative form association linking compatriots southern states provide marketing support ensure fair price produce eliminating middlemen multistate paddy farmers association proposes add 500 core paddy farmers thousands associate members paddy farmers kerala getting fair price produce cost labour expenses gone buy produce directly help marketing said nv giridharan president association based palakkad along alappuzha accounts bulk paddy production state giridharan said farmers southern states also benefit able sell rice kerala without middlemen present rice often sold palakkadan matta premium reddish rice variety kerala states may rice added red colour sell original variety said association sell different types rice five states according giridharan focus online sales hopes start supply starting palakkad registered right majority members kerala states said andhra pradesh tamil nadu major paddy producers southern india association least 100 farmer members five states member one share paying rs 1000 also get one voting right besides others take associate membership paying rs 100 associate members voting rights get dividend association hopes around 5000 members current support price rs 25 50 per kg kerala sufficient farmers meet expenses association share cost beneficial us besides get marketing support states said krishnankutty paddy farmer member association cultivating organic matta rice palakkad according giridharan retired nutritional scientist longterm objective association includes making agriculture financially viable occupation ensuring safe health consumers</td>\n",
       "      <td>2019-04-29 19:30:00+05:30</td>\n",
       "      <td>agriculture</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>Wheat import duty increased to 40%</td>\n",
       "      <td>new delhi government increased import duty wheat 40 30 bid boost offtake domestically produced grain decision announced friday comes backdrop governments plan sell wheat open market offload excess grain storage units food corporation india state agencies senior official said higher duty made import wheat especially cheaper grains russia unviable flour mills mills purchase overseas pushes price farmers get produce last year able sell 7 million tonnes wheat open sale millers year also likely sell much hike duty would ensure millers wheat purchase benefits indian farmers official told et flour miller said customs duty raised governments open sale wheat millers biscuit makers could affected government plans sell 10 million tonnes wheat 2 million tonnes rice open market oversupply domestic market due backtoback bumper production foodgrains put wheat prices pressure country year india likely produce 99 12 million tonnes grain government first imposed 10 import duty wheat march 2017 november year doubled 20 raised 30 may 2018 order restrict imports</td>\n",
       "      <td>2019-04-29 12:30:00+05:30</td>\n",
       "      <td>agriculture</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>Govt hikes customs duty on wheat to 40%</td>\n",
       "      <td>government hiked customs duty wheat 40 per cent 30 per cent curb imports protect domestic industry government wants restrict overseas purchase domestic prices wheat come pressure countrys wheat output expected scale record high year central board indirect taxes customs cbic notification hiked basic customs duty bcd wheat 40 per cent may last year government hiked bcd wheat 30 per cent 20 per cent government fixed wheats minimum support price msp price buys farmers rs 1840 per quintal rs 1735 year earlier part decision fix support price minimum 1 5 times production cost countrys wheat production season might cross 100 million tonnes alltime high output stood record 99 70 million tonnes 201718 crop year julyjune</td>\n",
       "      <td>2019-04-28 19:30:00+05:30</td>\n",
       "      <td>agriculture</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>India hikes wheat import duty to support local farmers</td>\n",
       "      <td>mumbai india raised import duty wheat 40 percent 30 percent government said late friday worlds 2 producer grain tries support local farmers step comes prime minister narendra modis party looks contain rural discontent due lower crop prices amid voting general election began april 11 ends may 19 local wheat prices fallen 11 percent 2019 due ample supply last years crop forecasts record output hike duty likely make imports wheat unviable flour mills even recent declines global prices potentially dragging global grain markets local wheat production higher government trying ensure prices remain support levels said harish galipelli head commodities currencies inditrade derivatives commodities mumbai india raised price buys newseason wheat local farmers 6 percent 1840 rupees per 100 kg 2019 government usually purchases quarter wheat farmers stateset prices build stocks run major food welfare programme indias wheat production rise 2 percent 2019 year record 99 12 million tonnes according estimates countrys agriculture department one wheat crop grown india year planting starting late october harvesting march government wheat stocks stood 17 million tonnes april 1 nearly 30 percent time year ago 40 percent import duty imports viable flour mills buy local crop said mumbaibased grain dealer global trading firm declined identified authorised speak media indian flour millers imported 1 65 million tonnes wheat 201718 fiscal year 5 7 million tonnes year shipments mainly australia russia ukraine</td>\n",
       "      <td>2019-04-27 18:30:00+05:30</td>\n",
       "      <td>agriculture</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                                   post_title  \\\n",
       "0  0.0         15th edition of Golden leaf India awards presented            \n",
       "1  1.0         Multi-state initiative in South India to help paddy farmers   \n",
       "2  2.0         Wheat import duty increased to 40%                            \n",
       "3  3.0         Govt hikes customs duty on wheat to 40%                       \n",
       "4  4.0         India hikes wheat import duty to support local farmers        \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      post_text  \\\n",
       "0  15th edition golden leaf india awards southern tea competition jointly organized united planters association southern india tea board ahmedabad saw 66 shortlisted teas qualified screening panel jury comprising renowned tasters major buyers arun kumar convenor competition said bombay trading corporation five awards kanan devan hills plantations company pvt ltd kdhp united nilgiri tea estates company ltd shared four awards devon plantations industries ltd joonktollee tea industries ltd harrisons malayalam limitedsbu b kodanad estate shared three awards harrisons malayalam limitedsbu two awards waterfall estate pvt ltd woodbriar group shared one award vigneshwar estate tea factory six awards bought leaf factory segment kdhp managing director k mathew abraham said awards company high range region recognition high quality standards maintained company safe sustainable agronomical practices kdhp largest plantation company south india 41 awards years munnar region facilities located besides indian market exports tea europe us russian asian countries                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
       "1  kochi group paddy farmers kerala taken initiative form association linking compatriots southern states provide marketing support ensure fair price produce eliminating middlemen multistate paddy farmers association proposes add 500 core paddy farmers thousands associate members paddy farmers kerala getting fair price produce cost labour expenses gone buy produce directly help marketing said nv giridharan president association based palakkad along alappuzha accounts bulk paddy production state giridharan said farmers southern states also benefit able sell rice kerala without middlemen present rice often sold palakkadan matta premium reddish rice variety kerala states may rice added red colour sell original variety said association sell different types rice five states according giridharan focus online sales hopes start supply starting palakkad registered right majority members kerala states said andhra pradesh tamil nadu major paddy producers southern india association least 100 farmer members five states member one share paying rs 1000 also get one voting right besides others take associate membership paying rs 100 associate members voting rights get dividend association hopes around 5000 members current support price rs 25 50 per kg kerala sufficient farmers meet expenses association share cost beneficial us besides get marketing support states said krishnankutty paddy farmer member association cultivating organic matta rice palakkad according giridharan retired nutritional scientist longterm objective association includes making agriculture financially viable occupation ensuring safe health consumers   \n",
       "2  new delhi government increased import duty wheat 40 30 bid boost offtake domestically produced grain decision announced friday comes backdrop governments plan sell wheat open market offload excess grain storage units food corporation india state agencies senior official said higher duty made import wheat especially cheaper grains russia unviable flour mills mills purchase overseas pushes price farmers get produce last year able sell 7 million tonnes wheat open sale millers year also likely sell much hike duty would ensure millers wheat purchase benefits indian farmers official told et flour miller said customs duty raised governments open sale wheat millers biscuit makers could affected government plans sell 10 million tonnes wheat 2 million tonnes rice open market oversupply domestic market due backtoback bumper production foodgrains put wheat prices pressure country year india likely produce 99 12 million tonnes grain government first imposed 10 import duty wheat march 2017 november year doubled 20 raised 30 may 2018 order restrict imports                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
       "3  government hiked customs duty wheat 40 per cent 30 per cent curb imports protect domestic industry government wants restrict overseas purchase domestic prices wheat come pressure countrys wheat output expected scale record high year central board indirect taxes customs cbic notification hiked basic customs duty bcd wheat 40 per cent may last year government hiked bcd wheat 30 per cent 20 per cent government fixed wheats minimum support price msp price buys farmers rs 1840 per quintal rs 1735 year earlier part decision fix support price minimum 1 5 times production cost countrys wheat production season might cross 100 million tonnes alltime high output stood record 99 70 million tonnes 201718 crop year julyjune                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
       "4  mumbai india raised import duty wheat 40 percent 30 percent government said late friday worlds 2 producer grain tries support local farmers step comes prime minister narendra modis party looks contain rural discontent due lower crop prices amid voting general election began april 11 ends may 19 local wheat prices fallen 11 percent 2019 due ample supply last years crop forecasts record output hike duty likely make imports wheat unviable flour mills even recent declines global prices potentially dragging global grain markets local wheat production higher government trying ensure prices remain support levels said harish galipelli head commodities currencies inditrade derivatives commodities mumbai india raised price buys newseason wheat local farmers 6 percent 1840 rupees per 100 kg 2019 government usually purchases quarter wheat farmers stateset prices build stocks run major food welfare programme indias wheat production rise 2 percent 2019 year record 99 12 million tonnes according estimates countrys agriculture department one wheat crop grown india year planting starting late october harvesting march government wheat stocks stood 17 million tonnes april 1 nearly 30 percent time year ago 40 percent import duty imports viable flour mills buy local crop said mumbaibased grain dealer global trading firm declined identified authorised speak media indian flour millers imported 1 65 million tonnes wheat 201718 fiscal year 5 7 million tonnes year shipments mainly australia russia ukraine                                                                                                                               \n",
       "\n",
       "                  post_date     category  \n",
       "0 2019-04-30 23:30:00+05:30  agriculture  \n",
       "1 2019-04-29 19:30:00+05:30  agriculture  \n",
       "2 2019-04-29 12:30:00+05:30  agriculture  \n",
       "3 2019-04-28 19:30:00+05:30  agriculture  \n",
       "4 2019-04-27 18:30:00+05:30  agriculture  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', -1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'post_title', 'post_text', 'post_date', 'category'], dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'15th edition golden leaf india awards southern tea competition jointly organized united planters association southern india tea board ahmedabad saw 66 shortlisted teas qualified screening panel jury comprising renowned tasters major buyers arun kumar convenor competition said bombay trading corporation five awards kanan devan hills plantations company pvt ltd kdhp united nilgiri tea estates company ltd shared four awards devon plantations industries ltd joonktollee tea industries ltd harrisons malayalam limitedsbu b kodanad estate shared three awards harrisons malayalam limitedsbu two awards waterfall estate pvt ltd woodbriar group shared one award vigneshwar estate tea factory six awards bought leaf factory segment kdhp managing director k mathew abraham said awards company high range region recognition high quality standards maintained company safe sustainable agronomical practices kdhp largest plantation company south india 41 awards years munnar region facilities located besides indian market exports tea europe us russian asian countries'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['post_text'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0    0\n",
       "post_title    0\n",
       "post_text     0\n",
       "post_date     0\n",
       "category      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "foreign-trade       48\n",
       "finance             47\n",
       "infrastructure      47\n",
       "agriculture         47\n",
       "corporate-trends    47\n",
       "policy              45\n",
       "indicators          43\n",
       "Name: category, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['category'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = df['post_text']  # this time we want to look at the text\n",
    "y = df['category']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(217, 10332)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "count_vect = CountVectorizer()\n",
    "\n",
    "X_train_counts = count_vect.fit_transform(X_train)\n",
    "X_train_counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(217, 10332)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "tfidf_transformer = TfidfTransformer()\n",
    "\n",
    "X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)\n",
    "X_train_tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(217, 10332)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "vectorizer = TfidfVectorizer()\n",
    "\n",
    "X_train_tfidf = vectorizer.fit_transform(X_train) # remember to use the original X_train set\n",
    "X_train_tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
       "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "     verbose=0)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "clf = LinearSVC()\n",
    "clf.fit(X_train_tfidf,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('tfidf', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.float64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), norm='l2', preprocessor=None, smooth_idf=True,...ax_iter=1000,\n",
       "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "     verbose=0))])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "# from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "# from sklearn.svm import LinearSVC\n",
    "\n",
    "text_clf = Pipeline([('tfidf', TfidfVectorizer()),\n",
    "                     ('clf', LinearSVC()),\n",
    "])\n",
    "\n",
    "# Feed the training data through the pipeline\n",
    "text_clf.fit(X_train, y_train)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Form a prediction set\n",
    "predictions = text_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[15  0  0  1  1  0  0]\n",
      " [ 0 11  1  0  0  0  4]\n",
      " [ 0  0  8  0  3  1  2]\n",
      " [ 0  1  0 16  1  0  0]\n",
      " [ 1  0  1  1 11  0  0]\n",
      " [ 0  1  0  0  0 12  0]\n",
      " [ 0  3  1  0  4  0  7]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "print(metrics.confusion_matrix(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                  precision    recall  f1-score   support\n",
      "\n",
      "     agriculture       0.94      0.88      0.91        17\n",
      "corporate-trends       0.69      0.69      0.69        16\n",
      "         finance       0.73      0.57      0.64        14\n",
      "   foreign-trade       0.89      0.89      0.89        18\n",
      "      indicators       0.55      0.79      0.65        14\n",
      "  infrastructure       0.92      0.92      0.92        13\n",
      "          policy       0.54      0.47      0.50        15\n",
      "\n",
      "       micro avg       0.75      0.75      0.75       107\n",
      "       macro avg       0.75      0.74      0.74       107\n",
      "    weighted avg       0.76      0.75      0.75       107\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print a classification report\n",
    "print(metrics.classification_report(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7476635514018691\n"
     ]
    }
   ],
   "source": [
    "print(metrics.accuracy_score(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "sid = SentimentIntensityAnalyzer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['scores'] = df['post_text'].apply(lambda text : sid.polarity_scores(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['compound'] = df['scores'].apply(lambda d : d['compound'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['comp_score'] = df['compound'].apply(lambda score: 'positive' if score >=0 else 'negative')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "positive    261\n",
       "negative    63 \n",
       "Name: comp_score, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['comp_score'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.decomposition import LatentDirichletAllocation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer(max_df=0.9,min_df=2,stop_words='english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtm = cv.fit_transform(df['post_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "LDA =  LatentDirichletAllocation(n_components=7,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LatentDirichletAllocation(batch_size=128, doc_topic_prior=None,\n",
       "             evaluate_every=-1, learning_decay=0.7,\n",
       "             learning_method='batch', learning_offset=10.0,\n",
       "             max_doc_update_iter=100, max_iter=10, mean_change_tol=0.001,\n",
       "             n_components=7, n_jobs=None, n_topics=None, perp_tol=0.1,\n",
       "             random_state=42, topic_word_prior=None,\n",
       "             total_samples=1000000.0, verbose=0)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LDA.fit(dtm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ambit', 'ambitions', 'ambitious', 'amc', 'amcs']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.get_feature_names()[500:505]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(LDA.components_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7, 5848)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LDA.components_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5848"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cv.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accessible\n",
      "maintains\n",
      "banking\n",
      "voted\n",
      "account\n",
      "offices\n",
      "monitors\n",
      "inherent\n",
      "pricesensitive\n",
      "investing\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "for i in range(10):\n",
    "    random_word_id = random.randint(0,5848)\n",
    "    print(cv.get_feature_names()[random_word_id])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "single_topic = LDA.components_[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4997, 1785, 2314, ..., 1489, 4159, 4634], dtype=int64)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the indices that would sort this array.\n",
    "single_topic.argsort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.14285714436971153"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Word least representative of this topic\n",
    "single_topic[1785]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "108.90128082791664"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Word most representative of this topic\n",
    "single_topic[4634]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3065, 1426, 2121, 3430, 4607, 4156, 4663, 1489, 4159, 4634],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Top 10 words for this topic:\n",
    "single_topic.argsort()[-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_word_indices = single_topic.argsort()[-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "km\n",
      "cost\n",
      "expressway\n",
      "minister\n",
      "road\n",
      "project\n",
      "said\n",
      "crore\n",
      "projects\n",
      "rs\n"
     ]
    }
   ],
   "source": [
    "for index in top_word_indices:\n",
    "    print(cv.get_feature_names()[index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "THE TOP 15 WORDS FOR TOPIC #0\n",
      "['gadkari', 'foundation', 'infrastructure', 'transport', 'new', 'delhi', 'highways', 'highway', 'construction', 'government', 'km', 'cost', 'expressway', 'minister', 'road', 'project', 'said', 'crore', 'projects', 'rs']\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "THE TOP 15 WORDS FOR TOPIC #1\n",
      "['steel', 'resolution', 'directors', 'mills', 'ilfs', 'lakh', 'independent', 'companies', 'cent', 'creditors', 'company', 'sugar', 'government', 'rs', 'india', 'production', 'year', 'million', 'said', 'tonnes']\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "THE TOP 15 WORDS FOR TOPIC #2\n",
      "['companies', 'year', 'banks', 'gdp', 'need', 'data', 'target', 'sector', 'lakh', 'cent', 'new', 'rbi', 'fiscal', 'bank', 'growth', 'india', 'government', 'crore', 'rs', 'said']\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "THE TOP 15 WORDS FOR TOPIC #3\n",
      "['court', 'million', 'report', 'money', 'department', 'new', 'company', 'group', 'banks', 'crore', 'india', 'government', 'scheme', 'companies', 'financial', 'information', 'rs', 'farmers', 'tax', 'said']\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "THE TOP 15 WORDS FOR TOPIC #4\n",
      "['farmer', 'bank', 'water', 'cities', 'spv', 'production', 'financial', 'government', 'kazakhstan', 'business', 'work', 'loans', 'said', 'assets', 'smart', 'city', 'india', 'crore', 'farmers', 'rs']\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "THE TOP 15 WORDS FOR TOPIC #5\n",
      "['order', 'company', 'cooperation', 'years', 'prices', 'goods', 'trade', 'high', 'new', 'credit', 'services', 'companies', 'year', 'gst', 'court', 'government', 'countries', 'india', 'tax', 'said']\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "THE TOP 15 WORDS FOR TOPIC #6\n",
      "['data', 'china', '2018', 'usd', 'march', 'lakh', 'years', 'report', 'indias', 'crore', 'exports', 'growth', 'rs', 'indian', 'trade', 'year', 'billion', 'cent', 'said', 'india']\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for index,topic in enumerate(LDA.components_):\n",
    "    print(f'THE TOP 15 WORDS FOR TOPIC #{index}')\n",
    "    print([cv.get_feature_names()[i] for i in topic.argsort()[-20:]])\n",
    "    print('\\n')\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attaching Discovered Topic Labels to Original Articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<324x5848 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 46158 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(324, 5848)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtm.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "324"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_results = LDA.transform(dtm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(324, 7)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_results.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.00152216, 0.99085753, 0.00152442, 0.00152239, 0.0015238 ,\n",
       "       0.00152419, 0.0015255 ])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_results[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.  , 0.99, 0.  , 0.  , 0.  , 0.  , 0.  ])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_results[0].round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_results[0].argmax()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This means that our model thinks that the first article belongs to topic #1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combining with Original Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>post_title</th>\n",
       "      <th>post_text</th>\n",
       "      <th>post_date</th>\n",
       "      <th>category</th>\n",
       "      <th>scores</th>\n",
       "      <th>compound</th>\n",
       "      <th>comp_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>15th edition of Golden leaf India awards presented</td>\n",
       "      <td>15th edition golden leaf india awards southern tea competition jointly organized united planters association southern india tea board ahmedabad saw 66 shortlisted teas qualified screening panel jury comprising renowned tasters major buyers arun kumar convenor competition said bombay trading corporation five awards kanan devan hills plantations company pvt ltd kdhp united nilgiri tea estates company ltd shared four awards devon plantations industries ltd joonktollee tea industries ltd harrisons malayalam limitedsbu b kodanad estate shared three awards harrisons malayalam limitedsbu two awards waterfall estate pvt ltd woodbriar group shared one award vigneshwar estate tea factory six awards bought leaf factory segment kdhp managing director k mathew abraham said awards company high range region recognition high quality standards maintained company safe sustainable agronomical practices kdhp largest plantation company south india 41 awards years munnar region facilities located besides indian market exports tea europe us russian asian countries</td>\n",
       "      <td>2019-04-30 23:30:00+05:30</td>\n",
       "      <td>agriculture</td>\n",
       "      <td>{'neg': 0.0, 'neu': 0.741, 'pos': 0.259, 'compound': 0.9909}</td>\n",
       "      <td>0.9909</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Multi-state initiative in South India to help paddy farmers</td>\n",
       "      <td>kochi group paddy farmers kerala taken initiative form association linking compatriots southern states provide marketing support ensure fair price produce eliminating middlemen multistate paddy farmers association proposes add 500 core paddy farmers thousands associate members paddy farmers kerala getting fair price produce cost labour expenses gone buy produce directly help marketing said nv giridharan president association based palakkad along alappuzha accounts bulk paddy production state giridharan said farmers southern states also benefit able sell rice kerala without middlemen present rice often sold palakkadan matta premium reddish rice variety kerala states may rice added red colour sell original variety said association sell different types rice five states according giridharan focus online sales hopes start supply starting palakkad registered right majority members kerala states said andhra pradesh tamil nadu major paddy producers southern india association least 100 farmer members five states member one share paying rs 1000 also get one voting right besides others take associate membership paying rs 100 associate members voting rights get dividend association hopes around 5000 members current support price rs 25 50 per kg kerala sufficient farmers meet expenses association share cost beneficial us besides get marketing support states said krishnankutty paddy farmer member association cultivating organic matta rice palakkad according giridharan retired nutritional scientist longterm objective association includes making agriculture financially viable occupation ensuring safe health consumers</td>\n",
       "      <td>2019-04-29 19:30:00+05:30</td>\n",
       "      <td>agriculture</td>\n",
       "      <td>{'neg': 0.0, 'neu': 0.833, 'pos': 0.167, 'compound': 0.9884}</td>\n",
       "      <td>0.9884</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>Wheat import duty increased to 40%</td>\n",
       "      <td>new delhi government increased import duty wheat 40 30 bid boost offtake domestically produced grain decision announced friday comes backdrop governments plan sell wheat open market offload excess grain storage units food corporation india state agencies senior official said higher duty made import wheat especially cheaper grains russia unviable flour mills mills purchase overseas pushes price farmers get produce last year able sell 7 million tonnes wheat open sale millers year also likely sell much hike duty would ensure millers wheat purchase benefits indian farmers official told et flour miller said customs duty raised governments open sale wheat millers biscuit makers could affected government plans sell 10 million tonnes wheat 2 million tonnes rice open market oversupply domestic market due backtoback bumper production foodgrains put wheat prices pressure country year india likely produce 99 12 million tonnes grain government first imposed 10 import duty wheat march 2017 november year doubled 20 raised 30 may 2018 order restrict imports</td>\n",
       "      <td>2019-04-29 12:30:00+05:30</td>\n",
       "      <td>agriculture</td>\n",
       "      <td>{'neg': 0.046, 'neu': 0.893, 'pos': 0.06, 'compound': 0.5106}</td>\n",
       "      <td>0.5106</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>Govt hikes customs duty on wheat to 40%</td>\n",
       "      <td>government hiked customs duty wheat 40 per cent 30 per cent curb imports protect domestic industry government wants restrict overseas purchase domestic prices wheat come pressure countrys wheat output expected scale record high year central board indirect taxes customs cbic notification hiked basic customs duty bcd wheat 40 per cent may last year government hiked bcd wheat 30 per cent 20 per cent government fixed wheats minimum support price msp price buys farmers rs 1840 per quintal rs 1735 year earlier part decision fix support price minimum 1 5 times production cost countrys wheat production season might cross 100 million tonnes alltime high output stood record 99 70 million tonnes 201718 crop year julyjune</td>\n",
       "      <td>2019-04-28 19:30:00+05:30</td>\n",
       "      <td>agriculture</td>\n",
       "      <td>{'neg': 0.04, 'neu': 0.893, 'pos': 0.067, 'compound': 0.4939}</td>\n",
       "      <td>0.4939</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>India hikes wheat import duty to support local farmers</td>\n",
       "      <td>mumbai india raised import duty wheat 40 percent 30 percent government said late friday worlds 2 producer grain tries support local farmers step comes prime minister narendra modis party looks contain rural discontent due lower crop prices amid voting general election began april 11 ends may 19 local wheat prices fallen 11 percent 2019 due ample supply last years crop forecasts record output hike duty likely make imports wheat unviable flour mills even recent declines global prices potentially dragging global grain markets local wheat production higher government trying ensure prices remain support levels said harish galipelli head commodities currencies inditrade derivatives commodities mumbai india raised price buys newseason wheat local farmers 6 percent 1840 rupees per 100 kg 2019 government usually purchases quarter wheat farmers stateset prices build stocks run major food welfare programme indias wheat production rise 2 percent 2019 year record 99 12 million tonnes according estimates countrys agriculture department one wheat crop grown india year planting starting late october harvesting march government wheat stocks stood 17 million tonnes april 1 nearly 30 percent time year ago 40 percent import duty imports viable flour mills buy local crop said mumbaibased grain dealer global trading firm declined identified authorised speak media indian flour millers imported 1 65 million tonnes wheat 201718 fiscal year 5 7 million tonnes year shipments mainly australia russia ukraine</td>\n",
       "      <td>2019-04-27 18:30:00+05:30</td>\n",
       "      <td>agriculture</td>\n",
       "      <td>{'neg': 0.021, 'neu': 0.932, 'pos': 0.047, 'compound': 0.7184}</td>\n",
       "      <td>0.7184</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                                   post_title  \\\n",
       "0  0.0         15th edition of Golden leaf India awards presented            \n",
       "1  1.0         Multi-state initiative in South India to help paddy farmers   \n",
       "2  2.0         Wheat import duty increased to 40%                            \n",
       "3  3.0         Govt hikes customs duty on wheat to 40%                       \n",
       "4  4.0         India hikes wheat import duty to support local farmers        \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      post_text  \\\n",
       "0  15th edition golden leaf india awards southern tea competition jointly organized united planters association southern india tea board ahmedabad saw 66 shortlisted teas qualified screening panel jury comprising renowned tasters major buyers arun kumar convenor competition said bombay trading corporation five awards kanan devan hills plantations company pvt ltd kdhp united nilgiri tea estates company ltd shared four awards devon plantations industries ltd joonktollee tea industries ltd harrisons malayalam limitedsbu b kodanad estate shared three awards harrisons malayalam limitedsbu two awards waterfall estate pvt ltd woodbriar group shared one award vigneshwar estate tea factory six awards bought leaf factory segment kdhp managing director k mathew abraham said awards company high range region recognition high quality standards maintained company safe sustainable agronomical practices kdhp largest plantation company south india 41 awards years munnar region facilities located besides indian market exports tea europe us russian asian countries                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
       "1  kochi group paddy farmers kerala taken initiative form association linking compatriots southern states provide marketing support ensure fair price produce eliminating middlemen multistate paddy farmers association proposes add 500 core paddy farmers thousands associate members paddy farmers kerala getting fair price produce cost labour expenses gone buy produce directly help marketing said nv giridharan president association based palakkad along alappuzha accounts bulk paddy production state giridharan said farmers southern states also benefit able sell rice kerala without middlemen present rice often sold palakkadan matta premium reddish rice variety kerala states may rice added red colour sell original variety said association sell different types rice five states according giridharan focus online sales hopes start supply starting palakkad registered right majority members kerala states said andhra pradesh tamil nadu major paddy producers southern india association least 100 farmer members five states member one share paying rs 1000 also get one voting right besides others take associate membership paying rs 100 associate members voting rights get dividend association hopes around 5000 members current support price rs 25 50 per kg kerala sufficient farmers meet expenses association share cost beneficial us besides get marketing support states said krishnankutty paddy farmer member association cultivating organic matta rice palakkad according giridharan retired nutritional scientist longterm objective association includes making agriculture financially viable occupation ensuring safe health consumers   \n",
       "2  new delhi government increased import duty wheat 40 30 bid boost offtake domestically produced grain decision announced friday comes backdrop governments plan sell wheat open market offload excess grain storage units food corporation india state agencies senior official said higher duty made import wheat especially cheaper grains russia unviable flour mills mills purchase overseas pushes price farmers get produce last year able sell 7 million tonnes wheat open sale millers year also likely sell much hike duty would ensure millers wheat purchase benefits indian farmers official told et flour miller said customs duty raised governments open sale wheat millers biscuit makers could affected government plans sell 10 million tonnes wheat 2 million tonnes rice open market oversupply domestic market due backtoback bumper production foodgrains put wheat prices pressure country year india likely produce 99 12 million tonnes grain government first imposed 10 import duty wheat march 2017 november year doubled 20 raised 30 may 2018 order restrict imports                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
       "3  government hiked customs duty wheat 40 per cent 30 per cent curb imports protect domestic industry government wants restrict overseas purchase domestic prices wheat come pressure countrys wheat output expected scale record high year central board indirect taxes customs cbic notification hiked basic customs duty bcd wheat 40 per cent may last year government hiked bcd wheat 30 per cent 20 per cent government fixed wheats minimum support price msp price buys farmers rs 1840 per quintal rs 1735 year earlier part decision fix support price minimum 1 5 times production cost countrys wheat production season might cross 100 million tonnes alltime high output stood record 99 70 million tonnes 201718 crop year julyjune                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
       "4  mumbai india raised import duty wheat 40 percent 30 percent government said late friday worlds 2 producer grain tries support local farmers step comes prime minister narendra modis party looks contain rural discontent due lower crop prices amid voting general election began april 11 ends may 19 local wheat prices fallen 11 percent 2019 due ample supply last years crop forecasts record output hike duty likely make imports wheat unviable flour mills even recent declines global prices potentially dragging global grain markets local wheat production higher government trying ensure prices remain support levels said harish galipelli head commodities currencies inditrade derivatives commodities mumbai india raised price buys newseason wheat local farmers 6 percent 1840 rupees per 100 kg 2019 government usually purchases quarter wheat farmers stateset prices build stocks run major food welfare programme indias wheat production rise 2 percent 2019 year record 99 12 million tonnes according estimates countrys agriculture department one wheat crop grown india year planting starting late october harvesting march government wheat stocks stood 17 million tonnes april 1 nearly 30 percent time year ago 40 percent import duty imports viable flour mills buy local crop said mumbaibased grain dealer global trading firm declined identified authorised speak media indian flour millers imported 1 65 million tonnes wheat 201718 fiscal year 5 7 million tonnes year shipments mainly australia russia ukraine                                                                                                                               \n",
       "\n",
       "                  post_date     category  \\\n",
       "0 2019-04-30 23:30:00+05:30  agriculture   \n",
       "1 2019-04-29 19:30:00+05:30  agriculture   \n",
       "2 2019-04-29 12:30:00+05:30  agriculture   \n",
       "3 2019-04-28 19:30:00+05:30  agriculture   \n",
       "4 2019-04-27 18:30:00+05:30  agriculture   \n",
       "\n",
       "                                                           scores  compound  \\\n",
       "0  {'neg': 0.0, 'neu': 0.741, 'pos': 0.259, 'compound': 0.9909}    0.9909     \n",
       "1  {'neg': 0.0, 'neu': 0.833, 'pos': 0.167, 'compound': 0.9884}    0.9884     \n",
       "2  {'neg': 0.046, 'neu': 0.893, 'pos': 0.06, 'compound': 0.5106}   0.5106     \n",
       "3  {'neg': 0.04, 'neu': 0.893, 'pos': 0.067, 'compound': 0.4939}   0.4939     \n",
       "4  {'neg': 0.021, 'neu': 0.932, 'pos': 0.047, 'compound': 0.7184}  0.7184     \n",
       "\n",
       "  comp_score  \n",
       "0  positive   \n",
       "1  positive   \n",
       "2  positive   \n",
       "3  positive   \n",
       "4  positive   "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 1, 1, 1, 2, 3, 2, 1, 6, 2, 5, 6, 4, 3, 1, 1, 6, 2, 6, 3, 4,\n",
       "       1, 5, 2, 2, 4, 1, 2, 3, 6, 4, 4, 2, 6, 6, 2, 2, 0, 1, 1, 1, 2, 4,\n",
       "       1, 4, 1, 6, 3, 3, 6, 3, 1, 6, 2, 6, 4, 6, 1, 5, 1, 2, 6, 6, 6, 4,\n",
       "       5, 6, 4, 3, 2, 6, 3, 1, 5, 2, 6, 2, 1, 5, 5, 2, 5, 2, 2, 6, 5, 2,\n",
       "       6, 2, 0, 6, 4, 5, 2, 5, 2, 6, 2, 2, 6, 6, 6, 1, 6, 2, 6, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 5, 2, 2, 2, 3, 2, 2, 6, 2, 3, 2, 6, 4, 2, 6, 2, 2,\n",
       "       2, 2, 2, 5, 2, 5, 6, 1, 6, 5, 6, 3, 2, 1, 1, 6, 2, 1, 3, 1, 6, 6,\n",
       "       2, 6, 0, 5, 2, 2, 6, 6, 6, 5, 6, 6, 6, 6, 1, 5, 5, 5, 2, 2, 1, 6,\n",
       "       6, 5, 6, 1, 2, 6, 6, 6, 4, 2, 5, 4, 2, 6, 6, 5, 6, 6, 5, 2, 5, 6,\n",
       "       2, 5, 6, 6, 2, 2, 6, 2, 2, 6, 2, 2, 5, 2, 2, 6, 2, 2, 2, 2, 2, 5,\n",
       "       2, 2, 2, 2, 6, 2, 2, 6, 2, 2, 2, 2, 2, 2, 6, 2, 0, 2, 4, 6, 6, 2,\n",
       "       1, 0, 0, 0, 6, 1, 6, 5, 6, 0, 0, 3, 0, 5, 6, 6, 0, 2, 0, 0, 6, 2,\n",
       "       0, 0, 3, 0, 2, 2, 0, 3, 0, 4, 6, 0, 4, 2, 6, 2, 4, 1, 4, 3, 6, 5,\n",
       "       2, 6, 5, 5, 3, 5, 5, 1, 2, 3, 5, 3, 2, 5, 2, 5, 5, 5, 6, 5, 5, 2,\n",
       "       5, 2, 1, 1, 1, 5, 5, 2, 4, 5, 5, 4, 6, 2, 2, 5], dtype=int64)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_results.argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Topic'] = topic_results.argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>post_title</th>\n",
       "      <th>post_text</th>\n",
       "      <th>post_date</th>\n",
       "      <th>category</th>\n",
       "      <th>scores</th>\n",
       "      <th>compound</th>\n",
       "      <th>comp_score</th>\n",
       "      <th>Topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>319</th>\n",
       "      <td>43.0</td>\n",
       "      <td>MNCs in India may face higher taxes</td>\n",
       "      <td>mumbai rules attributing profits multinational enterprises permanent establishment pe business connection india set significantly change draft report issued thursday night committee appointed central board direct taxes cbdt adopted according tax experts mncs carrying offshore operations india also sales revenue country may find higher profits attributed indian operations prescribed formuladriven approach result higher tax outgo foreign enterprise pe india general refers fixed place business defined tax treaties india tax business profits entity however tax imposed profits attributed operations carried india detailed accounts available determine attribution rule 10 empowers income tax officer indirect apportionment say percentage turnover however lack uniformity approach results litigation report cbdtappointed committee states appropriate attribute profits pe exclusively basis functions assets risks far alone game changer panels report outlines formula calculating profits attributed operations india giving equal weightage assigned sales manpower assets threefactor approach challenge significant manpower deployed india sales revenue also generated lot offshore centres india provide technical services overseas head office group companies could adversely impacted could result higher tax outgo says girish vanvari founder boutique advisory firm transactionsquare however report adds profits derived indian operations already subjected tax india say hands subsidiary deducted apportioned profits floor rate profits derived india set protect revenue interests committee suggests minimum 2 turnover derived deemed profits derived india certain circumstances say foreign enterprise incurring global losses also provides sale taken place india profits apportioned supply activities already taxed hands indian subsidiary may taxes payable foreign enterprise panel also considered emerging business models business models users contribute significantly profits foreign enterprise user base could taken account purpose profit attribution fourth factor apportionment addition sales manpower assets weightage range 1020 assigned user base points nangia advisors director sandeep jhunjhunwala</td>\n",
       "      <td>2019-04-19 15:30:00+05:30</td>\n",
       "      <td>policy</td>\n",
       "      <td>{'neg': 0.044, 'neu': 0.782, 'pos': 0.174, 'compound': 0.9891}</td>\n",
       "      <td>0.9891</td>\n",
       "      <td>positive</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>320</th>\n",
       "      <td>44.0</td>\n",
       "      <td>View: India’s growth numbers are not reflected by other indicators</td>\n",
       "      <td>kaushik basu paradoxical according latest government data indias gdp growth rate 6 6 though weak comparison indias performance last decade half global comparative terms commendable however terms several critical indicators performance economy outright disappointing latest growth number february 2019 core sector 2 1 index industrial production 0 1 export growth last five years zero rarely happened india 1991 indias savings investments rates crossed 30 2003 went 35 2008 making india look like fastgrowing economies east asia slid back 30 agriculture sector floundering finally india watchers concerned countrys jobs situation release official data stopped detailed studies azim premji university centre monitoring indian economy piecemeal evidence job announcements numbers applicants make clear situation grim square indias commendable growth indicators terms pure accounting explained growing inequality growth happening concentrated top bulk society working classes farmers small businesses barely inching forward come india important recognise several good policy initiatives also mistakes demonetisation november 2016 virtual consensus among experts around world blunder big negative effect small businesses workers farmers leaving rich virtually unaffected interestingly sharpest indictment comes finance ministrys economic survey 201617 lists countries similar sudden demonetisations since 1982 north korea venezuela myanmar iraq russia ussr ghana brazil cyprus list makes failure virtually selfexplanatory policy mistakes happen mistakes recognised corrective measures taken signals professionalism bolsters confidence nation lack expertise clear smaller initiatives new reservation policy holds 10 government jobs economically weak defined way seeming include everybody except richest 5 indias population however recognised one exclusion belonging scheduled castes disadvantaged castes cannot get jobs words bringing new reservation disadvantaged castes excluded 10 jobs earlier eligible baffling caveat one sector india natural advantage trade chinas wages rising able expand exports hugely would created jobs positive spillover agriculture sector export growth happen simply virtue cheap labour requires slew welldesigned policies among things requires management exchange rate india like market economies committed floating exchange rate system floating mean zero intervention countries including us occasionally use banks buy sell currency behalf central bank influence exchange rate thereby trade however complex matter intervention lump sum exchangerate sensitive onetime small batches lot research ought put use one initiative government supported effort cut bureaucratic costs going world banks index ease business india made significant progress rank improving 130th among 189 economies 2016 77th among 190 economies 2018 helped india create jobs boost exports retrospect clear happening instead making general effort cut bureaucratic costs disproportionate focus ten measures world bank tracks proxy measure nations bureaucratic cost reality many dimensions kind pure gaming system earned india pat back world bank much else closing let point economic policy mistakes cause short medium term hardships greater longrun worry india divisiveness groups propagating india home hindus various castes christians jews muslims parsis sikhs buddhists driving wedge upper caste hindus rest damage entire nations morale cause longrun damage growth india made remarkable morally resonant choice 1947 opted secular democracy easy early years know investment paying india among worlds fastest growing nations last decade half spread hatred divisiveness put end dream importantly behaviour damages moral fabric society know economist speak morality treated hardnosed enough must live wealth money alone uphold basic moral human ideals important individual nation writer chief economic advisor upa govt 2009 2012 chief economist world bank professor economics cornell university</td>\n",
       "      <td>2019-04-19 15:30:00+05:30</td>\n",
       "      <td>policy</td>\n",
       "      <td>{'neg': 0.108, 'neu': 0.713, 'pos': 0.179, 'compound': 0.9878}</td>\n",
       "      <td>0.9878</td>\n",
       "      <td>positive</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>321</th>\n",
       "      <td>45.0</td>\n",
       "      <td>RBI Governor Shaktikanta Das, Deputy Viral Acharya split vote on interest rates</td>\n",
       "      <td>mumbai reserve bank india governor shaktikanta das deputy viral acharya viewed data economic growth inflation different prisms led split voting interest rates last monetary policy review meeting raising possibility continued differences near future acharya chose wait incoming data would address several important uncertainties shape indian economy without mentioning leading speculation likely impact fiscal deficit inflation new government ends populist governor das felt need accelerate economic growth begun stutter acharya believes growth accelerating strong contribution services sector minutes monetary policy committee mpc meeting shows investment demand losing traction deceleration exports may impact investment activity das said statement forms part mpc minutes becomes necessary address challenges sustained growth indian economy deputy governor acharya sang different tune growth front signals domestic economy mixed said capacity utilisation continues improve augurs well future investment services growth remains robust mpc last meet april 4 cut interest rates quarter point 6 42 voting independent member chetan ghate acharya voted keep rates unchanged citing doubts sustainability low inflation questioning rbis projections inflation target 4 even dissent notes ghate acharya categorically ruled rate cuts prefer wait longer incremental data support rate action said shubhada rao chief economist yes bank inflation outlook looking benign headline inflation expected remain target current year said das acharya viewed state prices differently soft food inflation may persist long scenario elevated level inflation excluding food fuel would steer headline inflation away target rate 4 said acharya mpc system interest rate setting less three years old unanimity past seems giving way differing opinions even within rbi putting rest speculation insiders would vote unison</td>\n",
       "      <td>2019-04-20 00:30:00+05:30</td>\n",
       "      <td>policy</td>\n",
       "      <td>{'neg': 0.072, 'neu': 0.763, 'pos': 0.165, 'compound': 0.9783}</td>\n",
       "      <td>0.9783</td>\n",
       "      <td>positive</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322</th>\n",
       "      <td>46.0</td>\n",
       "      <td>50% local sourcing likely for companies to book EV ride under FAME-II</td>\n",
       "      <td>new delhi government insist electric vehicle makers source least 50 components locally want avail incentive programme bid orders even looks promote local manufacturing discourage imports interministerial steering committee led niti aayog chief executive officer amitabh kant decided companies meet 50 localisation threshold eligible incentives available faster adoption manufacturing hybrid electric vehicles fameii scheme boost electric mobility well make india initiative government use taxpayers money subsidise imports push 50 localisation content want promote local manufacturing kant told et steering committee held first meeting april 1 finalised missions approach make domestic industry competitive global level limitations twowheeler makers made mind would go ahead 50 localisation criteria several startups firms willing kant said panel held internal discussions big automobile manufacturers including hero motors bajaj auto electric twowheelers threewheelers tata motors mahindra mahindra electric buses twowheeler makers expressed inability meeting criteria 50 localisation citing limitations tata motors mm agreed localise electric buses government insiders told et government earmarked rs 10000 crore fameii roll one million electric buses country next three years besides electric twowheelers threewheelers part nearly twodozen decisions kantled committee also decided government adopt challenge route select states entrepreneurs set giga factories large scale manufacturing cells batteries country part governments twopronged strategy place india key driver global mobility revolution one boost domestic manufacturing insisting minimum 50 local content evehicles two create massive size scale electric vehicles country make investments made sector viable effect kant written states pool demand float tenders electric buses much municipalities fight get electric buses turn help clean cities money constraint fameii kant said government committed replenish allocate utilised interministerial committee national mission transformative mobility formed following cabinet decision promote clean sustainable mobility initiatives country consists secretary nine stakeholder ministries director general bureau indian standards members</td>\n",
       "      <td>2019-04-19 13:30:00+05:30</td>\n",
       "      <td>policy</td>\n",
       "      <td>{'neg': 0.03, 'neu': 0.821, 'pos': 0.148, 'compound': 0.9825}</td>\n",
       "      <td>0.9825</td>\n",
       "      <td>positive</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323</th>\n",
       "      <td>47.0</td>\n",
       "      <td>Protectionism doesn't really help preserve jobs: Raghuram Rajan</td>\n",
       "      <td>protectionism really help preserve jobs offers little defence jobdestroying effects automation artificial intelligence former reserve bank india governor raghuram rajan said asserting industrial developing nations cannot afford ignore democratic reaction left behind globalisation technological change delivering keynote address 2019 ecosoc forum financing development un headquarters monday rajan said open liberal democratic market system brought world enormous prosperity six decades second world war attack interestingly critics usual radical academics leftist leaders instead come prosperous nations world nations benefited tremendously open world order said rajan katherine dusak miller distinguished service professor finance university chicago booth school business said know protectionism really help preserve jobs competitive world jobs gained country protected sector often lost sectors rendered uncompetitive pay higher prices inputs added protectionism offers little defence job destroying effects automation artificial intelligence often larger source job losses guarantee redundancy help workforce stay ahead constant retraining populations age industrial countries become reliant foreign demand younger countries outside especially developing countries emerging markets boost growth said wise block imports today countries export future probably said rajan said nations recognise cost protectionism true cannot afford ignore democratic reaction left behind globalisation technological change true industrial countries developing countries pay attention left behind added concerns people addressed preserving open world start recognising globalisation trade investment flows disempowered people communities ecosoc forum financing development followup ffd forum annual platform promote consensus among key stakeholders financing sustainable development ministers senior un officials highlevel finance officials civil society business representatives local authorities meeting un headquarters fourday ffd forum run april 15 18 rajan told audience forum need preserve world open trade investment need keep democratic support order would argue must follow principle subsidiarity much strictly going forward decisions taken lowest level consistent effective governance decisions must taken idea cooperation must taken responsibly given spillovers country well rest world said given different impact globalisation across countries within countries rajan said must create room countries choose unique way coping countries decentralise power differentially affected communities chalk paths much developed country problem developing country problem conclude globalisation markets may paradoxically require far localization governance rajan stressed emerging markets developing countries take much responsibility fight keep world open chance succeeding though disparate effects globalisation technological change within across countries would managed much better said said may even need contain aspects globalisation order preserve open world many decades developing world told join global trading system open foreign direct investment recognise would affect people adversely pressed see inevitable cost development perhaps democracy still nascent countries implemented advice overriding domestic opposition wherever emerged said global trade investment global competition generally enhanced prosperity many countries rising tide lifted everyone studies show tradeaffected districts india incidents poverty relatively higher violent crime property crimes rajan stressed reality trade typically beneficial overall distributional impact creating winners losers implies work harder whether developing countries industrial countries help affected adjust however license protectionism unfortunately indeed see arising parts industrial world said</td>\n",
       "      <td>2019-04-17 19:30:00+05:30</td>\n",
       "      <td>policy</td>\n",
       "      <td>{'neg': 0.112, 'neu': 0.736, 'pos': 0.151, 'compound': 0.9594}</td>\n",
       "      <td>0.9594</td>\n",
       "      <td>positive</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0  \\\n",
       "319  43.0         \n",
       "320  44.0         \n",
       "321  45.0         \n",
       "322  46.0         \n",
       "323  47.0         \n",
       "\n",
       "                                                                          post_title  \\\n",
       "319  MNCs in India may face higher taxes                                               \n",
       "320  View: India’s growth numbers are not reflected by other indicators                \n",
       "321  RBI Governor Shaktikanta Das, Deputy Viral Acharya split vote on interest rates   \n",
       "322  50% local sourcing likely for companies to book EV ride under FAME-II             \n",
       "323  Protectionism doesn't really help preserve jobs: Raghuram Rajan                   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            post_text  \\\n",
       "319  mumbai rules attributing profits multinational enterprises permanent establishment pe business connection india set significantly change draft report issued thursday night committee appointed central board direct taxes cbdt adopted according tax experts mncs carrying offshore operations india also sales revenue country may find higher profits attributed indian operations prescribed formuladriven approach result higher tax outgo foreign enterprise pe india general refers fixed place business defined tax treaties india tax business profits entity however tax imposed profits attributed operations carried india detailed accounts available determine attribution rule 10 empowers income tax officer indirect apportionment say percentage turnover however lack uniformity approach results litigation report cbdtappointed committee states appropriate attribute profits pe exclusively basis functions assets risks far alone game changer panels report outlines formula calculating profits attributed operations india giving equal weightage assigned sales manpower assets threefactor approach challenge significant manpower deployed india sales revenue also generated lot offshore centres india provide technical services overseas head office group companies could adversely impacted could result higher tax outgo says girish vanvari founder boutique advisory firm transactionsquare however report adds profits derived indian operations already subjected tax india say hands subsidiary deducted apportioned profits floor rate profits derived india set protect revenue interests committee suggests minimum 2 turnover derived deemed profits derived india certain circumstances say foreign enterprise incurring global losses also provides sale taken place india profits apportioned supply activities already taxed hands indian subsidiary may taxes payable foreign enterprise panel also considered emerging business models business models users contribute significantly profits foreign enterprise user base could taken account purpose profit attribution fourth factor apportionment addition sales manpower assets weightage range 1020 assigned user base points nangia advisors director sandeep jhunjhunwala                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
       "320  kaushik basu paradoxical according latest government data indias gdp growth rate 6 6 though weak comparison indias performance last decade half global comparative terms commendable however terms several critical indicators performance economy outright disappointing latest growth number february 2019 core sector 2 1 index industrial production 0 1 export growth last five years zero rarely happened india 1991 indias savings investments rates crossed 30 2003 went 35 2008 making india look like fastgrowing economies east asia slid back 30 agriculture sector floundering finally india watchers concerned countrys jobs situation release official data stopped detailed studies azim premji university centre monitoring indian economy piecemeal evidence job announcements numbers applicants make clear situation grim square indias commendable growth indicators terms pure accounting explained growing inequality growth happening concentrated top bulk society working classes farmers small businesses barely inching forward come india important recognise several good policy initiatives also mistakes demonetisation november 2016 virtual consensus among experts around world blunder big negative effect small businesses workers farmers leaving rich virtually unaffected interestingly sharpest indictment comes finance ministrys economic survey 201617 lists countries similar sudden demonetisations since 1982 north korea venezuela myanmar iraq russia ussr ghana brazil cyprus list makes failure virtually selfexplanatory policy mistakes happen mistakes recognised corrective measures taken signals professionalism bolsters confidence nation lack expertise clear smaller initiatives new reservation policy holds 10 government jobs economically weak defined way seeming include everybody except richest 5 indias population however recognised one exclusion belonging scheduled castes disadvantaged castes cannot get jobs words bringing new reservation disadvantaged castes excluded 10 jobs earlier eligible baffling caveat one sector india natural advantage trade chinas wages rising able expand exports hugely would created jobs positive spillover agriculture sector export growth happen simply virtue cheap labour requires slew welldesigned policies among things requires management exchange rate india like market economies committed floating exchange rate system floating mean zero intervention countries including us occasionally use banks buy sell currency behalf central bank influence exchange rate thereby trade however complex matter intervention lump sum exchangerate sensitive onetime small batches lot research ought put use one initiative government supported effort cut bureaucratic costs going world banks index ease business india made significant progress rank improving 130th among 189 economies 2016 77th among 190 economies 2018 helped india create jobs boost exports retrospect clear happening instead making general effort cut bureaucratic costs disproportionate focus ten measures world bank tracks proxy measure nations bureaucratic cost reality many dimensions kind pure gaming system earned india pat back world bank much else closing let point economic policy mistakes cause short medium term hardships greater longrun worry india divisiveness groups propagating india home hindus various castes christians jews muslims parsis sikhs buddhists driving wedge upper caste hindus rest damage entire nations morale cause longrun damage growth india made remarkable morally resonant choice 1947 opted secular democracy easy early years know investment paying india among worlds fastest growing nations last decade half spread hatred divisiveness put end dream importantly behaviour damages moral fabric society know economist speak morality treated hardnosed enough must live wealth money alone uphold basic moral human ideals important individual nation writer chief economic advisor upa govt 2009 2012 chief economist world bank professor economics cornell university   \n",
       "321  mumbai reserve bank india governor shaktikanta das deputy viral acharya viewed data economic growth inflation different prisms led split voting interest rates last monetary policy review meeting raising possibility continued differences near future acharya chose wait incoming data would address several important uncertainties shape indian economy without mentioning leading speculation likely impact fiscal deficit inflation new government ends populist governor das felt need accelerate economic growth begun stutter acharya believes growth accelerating strong contribution services sector minutes monetary policy committee mpc meeting shows investment demand losing traction deceleration exports may impact investment activity das said statement forms part mpc minutes becomes necessary address challenges sustained growth indian economy deputy governor acharya sang different tune growth front signals domestic economy mixed said capacity utilisation continues improve augurs well future investment services growth remains robust mpc last meet april 4 cut interest rates quarter point 6 42 voting independent member chetan ghate acharya voted keep rates unchanged citing doubts sustainability low inflation questioning rbis projections inflation target 4 even dissent notes ghate acharya categorically ruled rate cuts prefer wait longer incremental data support rate action said shubhada rao chief economist yes bank inflation outlook looking benign headline inflation expected remain target current year said das acharya viewed state prices differently soft food inflation may persist long scenario elevated level inflation excluding food fuel would steer headline inflation away target rate 4 said acharya mpc system interest rate setting less three years old unanimity past seems giving way differing opinions even within rbi putting rest speculation insiders would vote unison                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      \n",
       "322  new delhi government insist electric vehicle makers source least 50 components locally want avail incentive programme bid orders even looks promote local manufacturing discourage imports interministerial steering committee led niti aayog chief executive officer amitabh kant decided companies meet 50 localisation threshold eligible incentives available faster adoption manufacturing hybrid electric vehicles fameii scheme boost electric mobility well make india initiative government use taxpayers money subsidise imports push 50 localisation content want promote local manufacturing kant told et steering committee held first meeting april 1 finalised missions approach make domestic industry competitive global level limitations twowheeler makers made mind would go ahead 50 localisation criteria several startups firms willing kant said panel held internal discussions big automobile manufacturers including hero motors bajaj auto electric twowheelers threewheelers tata motors mahindra mahindra electric buses twowheeler makers expressed inability meeting criteria 50 localisation citing limitations tata motors mm agreed localise electric buses government insiders told et government earmarked rs 10000 crore fameii roll one million electric buses country next three years besides electric twowheelers threewheelers part nearly twodozen decisions kantled committee also decided government adopt challenge route select states entrepreneurs set giga factories large scale manufacturing cells batteries country part governments twopronged strategy place india key driver global mobility revolution one boost domestic manufacturing insisting minimum 50 local content evehicles two create massive size scale electric vehicles country make investments made sector viable effect kant written states pool demand float tenders electric buses much municipalities fight get electric buses turn help clean cities money constraint fameii kant said government committed replenish allocate utilised interministerial committee national mission transformative mobility formed following cabinet decision promote clean sustainable mobility initiatives country consists secretary nine stakeholder ministries director general bureau indian standards members                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
       "323  protectionism really help preserve jobs offers little defence jobdestroying effects automation artificial intelligence former reserve bank india governor raghuram rajan said asserting industrial developing nations cannot afford ignore democratic reaction left behind globalisation technological change delivering keynote address 2019 ecosoc forum financing development un headquarters monday rajan said open liberal democratic market system brought world enormous prosperity six decades second world war attack interestingly critics usual radical academics leftist leaders instead come prosperous nations world nations benefited tremendously open world order said rajan katherine dusak miller distinguished service professor finance university chicago booth school business said know protectionism really help preserve jobs competitive world jobs gained country protected sector often lost sectors rendered uncompetitive pay higher prices inputs added protectionism offers little defence job destroying effects automation artificial intelligence often larger source job losses guarantee redundancy help workforce stay ahead constant retraining populations age industrial countries become reliant foreign demand younger countries outside especially developing countries emerging markets boost growth said wise block imports today countries export future probably said rajan said nations recognise cost protectionism true cannot afford ignore democratic reaction left behind globalisation technological change true industrial countries developing countries pay attention left behind added concerns people addressed preserving open world start recognising globalisation trade investment flows disempowered people communities ecosoc forum financing development followup ffd forum annual platform promote consensus among key stakeholders financing sustainable development ministers senior un officials highlevel finance officials civil society business representatives local authorities meeting un headquarters fourday ffd forum run april 15 18 rajan told audience forum need preserve world open trade investment need keep democratic support order would argue must follow principle subsidiarity much strictly going forward decisions taken lowest level consistent effective governance decisions must taken idea cooperation must taken responsibly given spillovers country well rest world said given different impact globalisation across countries within countries rajan said must create room countries choose unique way coping countries decentralise power differentially affected communities chalk paths much developed country problem developing country problem conclude globalisation markets may paradoxically require far localization governance rajan stressed emerging markets developing countries take much responsibility fight keep world open chance succeeding though disparate effects globalisation technological change within across countries would managed much better said said may even need contain aspects globalisation order preserve open world many decades developing world told join global trading system open foreign direct investment recognise would affect people adversely pressed see inevitable cost development perhaps democracy still nascent countries implemented advice overriding domestic opposition wherever emerged said global trade investment global competition generally enhanced prosperity many countries rising tide lifted everyone studies show tradeaffected districts india incidents poverty relatively higher violent crime property crimes rajan stressed reality trade typically beneficial overall distributional impact creating winners losers implies work harder whether developing countries industrial countries help affected adjust however license protectionism unfortunately indeed see arising parts industrial world said                                                                                                                                              \n",
       "\n",
       "                    post_date category  \\\n",
       "319 2019-04-19 15:30:00+05:30  policy    \n",
       "320 2019-04-19 15:30:00+05:30  policy    \n",
       "321 2019-04-20 00:30:00+05:30  policy    \n",
       "322 2019-04-19 13:30:00+05:30  policy    \n",
       "323 2019-04-17 19:30:00+05:30  policy    \n",
       "\n",
       "                                                             scores  compound  \\\n",
       "319  {'neg': 0.044, 'neu': 0.782, 'pos': 0.174, 'compound': 0.9891}  0.9891     \n",
       "320  {'neg': 0.108, 'neu': 0.713, 'pos': 0.179, 'compound': 0.9878}  0.9878     \n",
       "321  {'neg': 0.072, 'neu': 0.763, 'pos': 0.165, 'compound': 0.9783}  0.9783     \n",
       "322  {'neg': 0.03, 'neu': 0.821, 'pos': 0.148, 'compound': 0.9825}   0.9825     \n",
       "323  {'neg': 0.112, 'neu': 0.736, 'pos': 0.151, 'compound': 0.9594}  0.9594     \n",
       "\n",
       "    comp_score  Topic  \n",
       "319  positive   4      \n",
       "320  positive   6      \n",
       "321  positive   2      \n",
       "322  positive   2      \n",
       "323  positive   5      "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}