LDA2-vis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "71ccb5d5-6dcb-47e9-a9a4-6dd927ca7a56",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1: Import necessary libraries\n",
    "import gensim\n",
    "from gensim.corpora.dictionary import Dictionary\n",
    "from gensim.models import LdaModel\n",
    "from gensim.utils import simple_preprocess\n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "import pandas as pd\n",
    "import pyLDAvis\n",
    "import pyLDAvis.gensim_models as gensimvis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cedddb27-20fb-4987-8104-6c279895c462",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 2: Sample data\n",
    "documents = [\n",
    "    \"Natural language processing is a fascinating field of study.\",\n",
    "    \"Machine learning and deep learning are subsets of artificial intelligence.\",\n",
    "    \"Text mining involves the process of extracting meaningful information from text.\",\n",
    "    \"Topic modeling is a technique for discovering abstract topics within a collection of documents.\",\n",
    "    \"LDA is a popular topic modeling algorithm in the field of NLP.\",\n",
    "    \"Deep learning has revolutionized the field of artificial intelligence.\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9b1168ee-f888-41dd-9679-436914d93cc6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Admin\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Step 3: Download stopwords\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "75ade415-6291-42ab-9645-061c7df9f237",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 4: Define stop words\n",
    "stop_words = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c8c28065-3a1a-463c-92b7-5e04c45f7bf9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Processed Documents:\n",
      "Document 1: ['natural', 'language', 'processing', 'fascinating', 'field', 'study']\n",
      "Document 2: ['machine', 'learning', 'deep', 'learning', 'subsets', 'artificial', 'intelligence']\n",
      "Document 3: ['text', 'mining', 'involves', 'process', 'extracting', 'meaningful', 'information', 'text']\n",
      "Document 4: ['topic', 'modeling', 'technique', 'discovering', 'abstract', 'topics', 'within', 'collection', 'documents']\n",
      "Document 5: ['lda', 'popular', 'topic', 'modeling', 'algorithm', 'field', 'nlp']\n",
      "Document 6: ['deep', 'learning', 'revolutionized', 'field', 'artificial', 'intelligence']\n"
     ]
    }
   ],
   "source": [
    "# Step 5: Preprocess the Text\n",
    "def preprocess(text):\n",
    "    tokens = simple_preprocess(text, deacc=True)  # Tokenization and normalization\n",
    "    tokens = [token for token in tokens if token not in stop_words]  # Stop word removal\n",
    "    return tokens\n",
    "\n",
    "# Apply preprocessing to each document\n",
    "processed_docs = [preprocess(doc) for doc in documents]\n",
    "\n",
    "# Print processed documents\n",
    "print(\"\\nProcessed Documents:\")\n",
    "for i, doc in enumerate(processed_docs):\n",
    "    print(f\"Document {i + 1}: {doc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "96b3fb61-33e1-430a-afdb-e22eef98ffaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 6: Create a dictionary and corpus\n",
    "dictionary = Dictionary(processed_docs)\n",
    "corpus = [dictionary.doc2bow(doc) for doc in processed_docs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fcda2c84-3e9c-4298-ba27-03fc12bd4208",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Dictionary:\n",
      "{'fascinating': 0, 'field': 1, 'language': 2, 'natural': 3, 'processing': 4, 'study': 5, 'artificial': 6, 'deep': 7, 'intelligence': 8, 'learning': 9, 'machine': 10, 'subsets': 11, 'extracting': 12, 'information': 13, 'involves': 14, 'meaningful': 15, 'mining': 16, 'process': 17, 'text': 18, 'abstract': 19, 'collection': 20, 'discovering': 21, 'documents': 22, 'modeling': 23, 'technique': 24, 'topic': 25, 'topics': 26, 'within': 27, 'algorithm': 28, 'lda': 29, 'nlp': 30, 'popular': 31, 'revolutionized': 32}\n"
     ]
    }
   ],
   "source": [
    "# Display dictionary\n",
    "print(\"\\nDictionary:\")\n",
    "print(dictionary.token2id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bb4b8328-2464-4b4f-b0f7-bfaae0d3e1e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Corpus:\n",
      "Document 1: [(0, 1), (1, 1), (2, 1), (3, 1), (4, 1), (5, 1)]\n",
      "Document 2: [(6, 1), (7, 1), (8, 1), (9, 2), (10, 1), (11, 1)]\n",
      "Document 3: [(12, 1), (13, 1), (14, 1), (15, 1), (16, 1), (17, 1), (18, 2)]\n",
      "Document 4: [(19, 1), (20, 1), (21, 1), (22, 1), (23, 1), (24, 1), (25, 1), (26, 1), (27, 1)]\n",
      "Document 5: [(1, 1), (23, 1), (25, 1), (28, 1), (29, 1), (30, 1), (31, 1)]\n",
      "Document 6: [(1, 1), (6, 1), (7, 1), (8, 1), (9, 1), (32, 1)]\n"
     ]
    }
   ],
   "source": [
    "# Display corpus\n",
    "print(\"\\nCorpus:\")\n",
    "for doc_id, doc in enumerate(corpus):\n",
    "    print(f\"Document {doc_id + 1}: {doc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a80e10e6-e854-4813-a0dd-449523335446",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 7: Build the LDA model\n",
    "num_topics = 3\n",
    "lda_model = LdaModel(corpus=corpus, id2word=dictionary, num_topics=num_topics, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c73813e2-75ee-4712-9503-958382da4b75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Topic-Word Distribution:\n",
      "Topic 0: {'field': 0.07617569, 'processing': 0.075079665, 'fascinating': 0.07493144, 'study': 0.07486473, 'language': 0.07442345, 'natural': 0.07428721, 'text': 0.022472566, 'extracting': 0.0217278, 'involves': 0.021021795, 'process': 0.020985322}\n",
      "Topic 1: {'learning': 0.09580206, 'intelligence': 0.0667359, 'deep': 0.06666222, 'artificial': 0.06646762, 'field': 0.039246384, 'topic': 0.038355812, 'machine': 0.03829403, 'subsets': 0.038262185, 'abstract': 0.038261633, 'revolutionized': 0.03822168}\n",
      "Topic 2: {'text': 0.07931718, 'modeling': 0.05485446, 'field': 0.054565594, 'topic': 0.054432344, 'nlp': 0.053706195, 'algorithm': 0.05359196, 'popular': 0.053562265, 'lda': 0.053146146, 'meaningful': 0.046936154, 'process': 0.0456014}\n"
     ]
    }
   ],
   "source": [
    "# Step 8: Topic-word distribution\n",
    "print(\"\\nTopic-Word Distribution:\")\n",
    "topics = lda_model.show_topics(formatted=False)\n",
    "topic_word_dist = {}\n",
    "for topic_num, words in topics:\n",
    "    topic_word_dist[topic_num] = {word: prob for word, prob in words}\n",
    "    print(f\"Topic {topic_num}: {topic_word_dist[topic_num]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f1b2dc09-4423-42d0-8555-7770fb2bd7ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Document-Topic Distribution:\n",
      "Document 0: [(0, 0.90296334), (1, 0.04835667), (2, 0.048679963)]\n",
      "Document 1: [(0, 0.04207211), (1, 0.9159677), (2, 0.041960176)]\n",
      "Document 2: [(0, 0.037648376), (1, 0.038033746), (2, 0.9243179)]\n",
      "Document 3: [(0, 0.03388926), (1, 0.93064), (2, 0.03547071)]\n",
      "Document 4: [(0, 0.043428745), (1, 0.043789327), (2, 0.9127819)]\n",
      "Document 5: [(0, 0.050577495), (1, 0.89983165), (2, 0.04959086)]\n"
     ]
    }
   ],
   "source": [
    "# Step 9: Document-topic distribution\n",
    "print(\"\\nDocument-Topic Distribution:\")\n",
    "doc_topic_dist = []\n",
    "for i, doc in enumerate(corpus):\n",
    "    doc_topics = lda_model.get_document_topics(doc)\n",
    "    doc_topic_dist.append(doc_topics)\n",
    "    print(f\"Document {i}: {doc_topics}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7ec3aa02-ee58-4178-8f0e-f8f377590733",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Conditional Probabilities for each word in each document:\n",
      "Document 0: [('fascinating', [(0, 0.050226413), (1, 0.0015088858), (2, 0.002024885)]), ('field', [(0, 0.05145469), (1, 0.02625679), (2, 0.036553912)]), ('language', [(0, 0.049725533), (1, 0.0015475627), (2, 0.0021446473)]), ('natural', [(0, 0.04959125), (1, 0.0014811677), (2, 0.0022877285)]), ('processing', [(0, 0.050372623), (1, 0.0014498696), (2, 0.0020572364)]), ('study', [(0, 0.050160613), (1, 0.0014648661), (2, 0.0021093497)])]\n",
      "Document 1: [('artificial', [(0, 0.0032517237), (1, 0.053283826), (2, 0.0025308228)]), ('deep', [(0, 0.0032583605), (1, 0.05347902), (2, 0.0023859714)]), ('intelligence', [(0, 0.003171573), (1, 0.05355291), (2, 0.0023964662)]), ('learning', [(0, 0.0032791658), (1, 0.08282172), (2, 0.002440297)]), ('machine', [(0, 0.0028751972), (1, 0.025329415), (2, 0.0020620103)]), ('subsets', [(0, 0.00291496), (1, 0.02529845), (2, 0.0020549772)])]\n",
      "Document 2: [('extracting', [(0, 0.003881533), (1, 0.0070027392), (2, 0.023279727)]), ('information', [(0, 0.0034226421), (1, 0.004956923), (2, 0.027385382)]), ('involves', [(0, 0.0035066293), (1, 0.005322434), (2, 0.026617391)]), ('meaningful', [(0, 0.0033233296), (1, 0.004092852), (2, 0.029137371)]), ('mining', [(0, 0.0034564873), (1, 0.0056572296), (2, 0.026094185)]), ('process', [(0, 0.0034876957), (1, 0.004654134), (2, 0.027854493)]), ('text', [(0, 0.0042938236), (1, 0.008177531), (2, 0.061158616)])]\n",
      "Document 3: [('abstract', [(0, 0.0027858794), (1, 0.02529791), (2, 0.0021486087)]), ('collection', [(0, 0.0028931666), (1, 0.025177125), (2, 0.0021545654)]), ('discovering', [(0, 0.0028066211), (1, 0.025066886), (2, 0.0022968652)]), ('documents', [(0, 0.0027730118), (1, 0.025180608), (2, 0.0022408764)]), ('modeling', [(0, 0.0034035973), (1, 0.025077773), (2, 0.036837023)]), ('technique', [(0, 0.0029218148), (1, 0.02486184), (2, 0.0023584159)]), ('topic', [(0, 0.003383456), (1, 0.02538952), (2, 0.036423355)]), ('topics', [(0, 0.0028501544), (1, 0.025128094), (2, 0.0022205866)]), ('within', [(0, 0.002928936), (1, 0.024737328), (2, 0.0024443557)])]\n",
      "Document 4: [('field', [(0, 0.05145469), (1, 0.02625679), (2, 0.036553912)]), ('modeling', [(0, 0.0034035973), (1, 0.025077773), (2, 0.036837023)]), ('topic', [(0, 0.003383456), (1, 0.02538952), (2, 0.036423355)]), ('algorithm', [(0, 0.003024159), (1, 0.0014496029), (2, 0.035600755)]), ('lda', [(0, 0.003259116), (1, 0.0014853596), (2, 0.035164893)]), ('nlp', [(0, 0.0029774215), (1, 0.0014344788), (2, 0.035712503)]), ('popular', [(0, 0.003016415), (1, 0.0014633965), (2, 0.0355717)])]\n",
      "Document 5: [('field', [(0, 0.05145469), (1, 0.02625679), (2, 0.036553912)]), ('artificial', [(0, 0.0032517237), (1, 0.053283826), (2, 0.0025308228)]), ('deep', [(0, 0.0032583605), (1, 0.05347902), (2, 0.0023859714)]), ('intelligence', [(0, 0.003171573), (1, 0.05355291), (2, 0.0023964662)]), ('learning', [(0, 0.0032791658), (1, 0.08282172), (2, 0.002440297)]), ('revolutionized', [(0, 0.0029124727), (1, 0.025259055), (2, 0.0020837951)])]\n"
     ]
    }
   ],
   "source": [
    "# Step 10: Conditional probabilities for each word in each document\n",
    "print(\"\\nConditional Probabilities for each word in each document:\")\n",
    "conditional_probs = []\n",
    "for i, doc in enumerate(corpus):\n",
    "    doc_conditional_probs = []\n",
    "    for word_id, freq in doc:\n",
    "        word = dictionary[word_id]\n",
    "        word_conditional_probs = []\n",
    "        for topic_num in range(num_topics):\n",
    "            prob = lda_model.get_term_topics(word_id, minimum_probability=0)[topic_num][1]\n",
    "            word_conditional_probs.append((topic_num, prob))\n",
    "        doc_conditional_probs.append((word, word_conditional_probs))\n",
    "    conditional_probs.append(doc_conditional_probs)\n",
    "    print(f\"Document {i}: {doc_conditional_probs}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "eedc794c-5319-4310-b8a1-ce6fb48ed8a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Topic Distribution for each document:\n",
      "Document 0: [(0, 0.9029644), (1, 0.048356406), (2, 0.048679125)]\n",
      "Document 1: [(0, 0.04207321), (1, 0.9159666), (2, 0.04196016)]\n",
      "Document 2: [(0, 0.037648395), (1, 0.03803528), (2, 0.92431635)]\n",
      "Document 3: [(0, 0.033889264), (1, 0.9306397), (2, 0.035471074)]\n",
      "Document 4: [(0, 0.04342796), (1, 0.043788128), (2, 0.9127839)]\n",
      "Document 5: [(0, 0.050551485), (1, 0.89986473), (2, 0.049583796)]\n"
     ]
    }
   ],
   "source": [
    "# Step 11: Topic distribution for each document\n",
    "print(\"\\nTopic Distribution for each document:\")\n",
    "for i, doc in enumerate(corpus):\n",
    "    doc_topic_dist = lda_model.get_document_topics(doc)\n",
    "    print(f\"Document {i}: {doc_topic_dist}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "504d71ab-f1d4-4702-a75a-515bf0d50174",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Topic-Word Distribution (Tabular):\n",
      "                       0         1         2\n",
      "field           0.076176  0.039246  0.054566\n",
      "processing      0.075080  0.000000  0.000000\n",
      "fascinating     0.074931  0.000000  0.000000\n",
      "study           0.074865  0.000000  0.000000\n",
      "language        0.074423  0.000000  0.000000\n",
      "natural         0.074287  0.000000  0.000000\n",
      "text            0.022473  0.000000  0.079317\n",
      "extracting      0.021728  0.000000  0.000000\n",
      "involves        0.021022  0.000000  0.000000\n",
      "process         0.020985  0.000000  0.045601\n",
      "learning        0.000000  0.095802  0.000000\n",
      "intelligence    0.000000  0.066736  0.000000\n",
      "deep            0.000000  0.066662  0.000000\n",
      "artificial      0.000000  0.066468  0.000000\n",
      "topic           0.000000  0.038356  0.054432\n",
      "machine         0.000000  0.038294  0.000000\n",
      "subsets         0.000000  0.038262  0.000000\n",
      "abstract        0.000000  0.038262  0.000000\n",
      "revolutionized  0.000000  0.038222  0.000000\n",
      "modeling        0.000000  0.000000  0.054854\n",
      "nlp             0.000000  0.000000  0.053706\n",
      "algorithm       0.000000  0.000000  0.053592\n",
      "popular         0.000000  0.000000  0.053562\n",
      "lda             0.000000  0.000000  0.053146\n",
      "meaningful      0.000000  0.000000  0.046936\n"
     ]
    }
   ],
   "source": [
    "# Step 12: Display the results in a tabular format using pandas\n",
    "# Topic-word distribution\n",
    "topic_word_df = pd.DataFrame(topic_word_dist).fillna(0)\n",
    "print(\"\\nTopic-Word Distribution (Tabular):\")\n",
    "print(topic_word_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5842f6b4-8c68-451f-b5e3-08f9903a8aa2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Document-Topic Distribution (Tabular):\n",
      "    Topic 0   Topic 1   Topic 2\n",
      "0  0.902964  0.048357  0.048679\n",
      "1  0.042072  0.915968  0.041960\n",
      "2  0.037648  0.038036  0.924316\n",
      "3  0.033889  0.930617  0.035494\n",
      "4  0.043429  0.043789  0.912782\n",
      "5  0.050550  0.899866  0.049584\n"
     ]
    }
   ],
   "source": [
    "# Document-topic distribution\n",
    "doc_topic_df = pd.DataFrame([[prob for _, prob in lda_model.get_document_topics(doc, minimum_probability=0)] for doc in corpus])\n",
    "doc_topic_df.columns = [f'Topic {i}' for i in range(num_topics)]\n",
    "print(\"\\nDocument-Topic Distribution (Tabular):\")\n",
    "print(doc_topic_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c1d2d3d7-2199-47a1-b9ef-61db03987cd1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Conditional Probabilities (Tabular):\n",
      "    Document            Word    Topic  Probability\n",
      "0      Doc 0     fascinating  Topic 0     0.050226\n",
      "1      Doc 0     fascinating  Topic 1     0.001509\n",
      "2      Doc 0     fascinating  Topic 2     0.002025\n",
      "3      Doc 0           field  Topic 0     0.051455\n",
      "4      Doc 0           field  Topic 1     0.026257\n",
      "..       ...             ...      ...          ...\n",
      "118    Doc 5        learning  Topic 1     0.082822\n",
      "119    Doc 5        learning  Topic 2     0.002440\n",
      "120    Doc 5  revolutionized  Topic 0     0.002912\n",
      "121    Doc 5  revolutionized  Topic 1     0.025259\n",
      "122    Doc 5  revolutionized  Topic 2     0.002084\n",
      "\n",
      "[123 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "# Conditional probabilities for each word in each document\n",
    "conditional_probs_df = []\n",
    "for i, doc in enumerate(conditional_probs):\n",
    "    for word, probs in doc:\n",
    "        for topic_num, prob in probs:\n",
    "            conditional_probs_df.append((f'Doc {i}', word, f'Topic {topic_num}', prob))\n",
    "conditional_probs_df = pd.DataFrame(conditional_probs_df, columns=['Document', 'Word', 'Topic', 'Probability'])\n",
    "print(\"\\nConditional Probabilities (Tabular):\")\n",
    "print(conditional_probs_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0035354e-cedb-464e-8fee-5abb35f6eeeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 13: Visualize the topics using pyLDAvis\n",
    "# Prepare the pyLDAvis data\n",
    "vis_data = gensimvis.prepare(lda_model, corpus, dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cebd6a19-5432-4476-ac63-f0f49f776353",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the visualization to an HTML file\n",
    "pyLDAvis.save_html(vis_data, 'lda_visualization.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0f8d4d3f-1d8f-4d94-b416-4fe499e840c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.jsdelivr.net/gh/bmabey/pyLDAvis@3.4.0/pyLDAvis/js/ldavis.v1.0.0.css\">\n",
       "\n",
       "\n",
       "<div id=\"ldavis_el16322205661301664798383600\" style=\"background-color:white;\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "\n",
       "var ldavis_el16322205661301664798383600_data = {\"mdsDat\": {\"x\": [-0.07990326890197423, 0.051600699818165924, 0.028302569083808432], \"y\": [-0.009925252703552032, -0.04609684348730732, 0.056022096190859356], \"topics\": [1, 2, 3], \"cluster\": [1, 1, 1], \"Freq\": [49.04121231125234, 34.85233202314916, 16.10645566559851]}, \"tinfo\": {\"Term\": [\"processing\", \"fascinating\", \"study\", \"language\", \"natural\", \"field\", \"text\", \"learning\", \"nlp\", \"algorithm\", \"popular\", \"lda\", \"meaningful\", \"process\", \"information\", \"involves\", \"mining\", \"deep\", \"intelligence\", \"artificial\", \"extracting\", \"modeling\", \"topic\", \"subsets\", \"revolutionized\", \"machine\", \"collection\", \"abstract\", \"topics\", \"documents\", \"learning\", \"intelligence\", \"deep\", \"artificial\", \"machine\", \"subsets\", \"abstract\", \"revolutionized\", \"collection\", \"documents\", \"topics\", \"discovering\", \"technique\", \"within\", \"topic\", \"modeling\", \"field\", \"extracting\", \"mining\", \"involves\", \"information\", \"process\", \"meaningful\", \"text\", \"language\", \"fascinating\", \"natural\", \"study\", \"processing\", \"lda\", \"popular\", \"nlp\", \"algorithm\", \"popular\", \"lda\", \"text\", \"meaningful\", \"process\", \"information\", \"involves\", \"mining\", \"extracting\", \"modeling\", \"topic\", \"field\", \"natural\", \"language\", \"study\", \"processing\", \"fascinating\", \"within\", \"technique\", \"discovering\", \"documents\", \"topics\", \"collection\", \"abstract\", \"revolutionized\", \"machine\", \"subsets\", \"artificial\", \"intelligence\", \"deep\", \"learning\", \"processing\", \"fascinating\", \"study\", \"language\", \"natural\", \"field\", \"extracting\", \"involves\", \"process\", \"mining\", \"information\", \"meaningful\", \"lda\", \"algorithm\", \"popular\", \"nlp\", \"within\", \"technique\", \"subsets\", \"revolutionized\", \"collection\", \"machine\", \"topics\", \"discovering\", \"abstract\", \"documents\", \"text\", \"modeling\", \"topic\", \"deep\", \"artificial\", \"intelligence\", \"learning\"], \"Freq\": [0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0202471491933656, 1.4073080667266027, 1.4057543489195639, 1.4016506422351616, 0.8075337928259947, 0.8068622800433791, 0.8068506534767171, 0.8060081201831336, 0.8042309051993818, 0.8043066349984508, 0.8031677813707511, 0.801839603245379, 0.7973895348554841, 0.7946860438750291, 0.8088366753130884, 0.8020758267991137, 0.8276168013454759, 0.38695613794581685, 0.3509979560649763, 0.3417234124155159, 0.33141379010160277, 0.32270938028693785, 0.3061145185213043, 0.41703512643192614, 0.21697257139204482, 0.21527913839035423, 0.21405371397575204, 0.21332821228814677, 0.21265772075800832, 0.2142396997634032, 0.2132626557349071, 0.8048680360280397, 0.8031560375082175, 0.8027110239683022, 0.7964748606986898, 1.1886871616739034, 0.7034088011333766, 0.6834055234663389, 0.6760699182645281, 0.6640349008531614, 0.6558165821800672, 0.6113069096566768, 0.8220765103067748, 0.8157504622298278, 0.8177474123376225, 0.2196499823411913, 0.21527409960896668, 0.21417507639481503, 0.21253731658465308, 0.2115114434132406, 0.22430612701926647, 0.2217677465626952, 0.21992523362775102, 0.21823055607778935, 0.21761190016395338, 0.21558149448423625, 0.2153969235692996, 0.21337423625890795, 0.21268812486776642, 0.2124660089013189, 0.22682203627400413, 0.2228964150145454, 0.22258590766846287, 0.2241871552649622, 0.5199849296618961, 0.5189583783463692, 0.5184963425325948, 0.5154401154633261, 0.514496590239419, 0.5275757539215675, 0.1504818756013515, 0.14559223737714533, 0.1453396373305316, 0.1449219784209169, 0.14446721835538262, 0.14312155323805378, 0.14224225842281263, 0.138958883525508, 0.13884871528087897, 0.13829253334891922, 0.13759639646543953, 0.13749371037325603, 0.13739477825755814, 0.13735886392581959, 0.1370795474348702, 0.13681880732240626, 0.13645425879489198, 0.1358170245115266, 0.1355118816943773, 0.1353220414264662, 0.15563995539493541, 0.14421045152387252, 0.1439382817785349, 0.14223191241201724, 0.14214059145139324, 0.14103137525160603, 0.1425175758322825], \"Total\": [0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.3869518802906105, 1.771235856992754, 1.770572169000044, 1.7706132699605588, 1.1570407250161674, 1.1567230672022561, 1.1577594587403939, 1.156741220367861, 1.1568919471184884, 1.1578592325027064, 1.1572339403295964, 1.1575818613846567, 1.1566509917914354, 1.156588567359735, 1.768525419321451, 1.768362788629761, 2.172939967604666, 1.1487449232038451, 1.1517365166659603, 1.1513505506458226, 1.1519509267215133, 1.1514545410838084, 1.1526448728927348, 1.761362243500765, 0.9476867864643377, 0.945748960149964, 0.9482002865563623, 0.9459996312155566, 0.9451799670045575, 1.1529568188849058, 1.1548223949840883, 1.1551268432895783, 1.1547606027961868, 1.1548223949840883, 1.1529568188849058, 1.761362243500765, 1.1526448728927348, 1.1514545410838084, 1.1519509267215133, 1.1513505506458226, 1.1517365166659603, 1.1487449232038451, 1.768362788629761, 1.768525419321451, 2.172939967604666, 0.9482002865563623, 0.9476867864643377, 0.9459996312155566, 0.9451799670045575, 0.945748960149964, 1.156588567359735, 1.1566509917914354, 1.1575818613846567, 1.1578592325027064, 1.1572339403295964, 1.1568919471184884, 1.1577594587403939, 1.156741220367861, 1.1570407250161674, 1.1567230672022561, 1.7706132699605588, 1.771235856992754, 1.770572169000044, 2.3869518802906105, 0.9451799670045575, 0.945748960149964, 0.9459996312155566, 0.9476867864643377, 0.9482002865563623, 2.172939967604666, 1.1487449232038451, 1.1513505506458226, 1.1514545410838084, 1.1517365166659603, 1.1519509267215133, 1.1526448728927348, 1.1529568188849058, 1.1547606027961868, 1.1548223949840883, 1.1551268432895783, 1.156588567359735, 1.1566509917914354, 1.1567230672022561, 1.156741220367861, 1.1568919471184884, 1.1570407250161674, 1.1572339403295964, 1.1575818613846567, 1.1577594587403939, 1.1578592325027064, 1.761362243500765, 1.768362788629761, 1.768525419321451, 1.770572169000044, 1.7706132699605588, 1.771235856992754, 2.3869518802906105], \"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\"], \"logprob\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -2.3455, -2.707, -2.7081, -2.711, -3.2625, -3.2633, -3.2633, -3.2644, -3.2666, -3.2665, -3.2679, -3.2695, -3.2751, -3.2785, -3.2608, -3.2692, -3.2379, -3.9981, -4.0957, -4.1224, -4.1531, -4.1797, -4.2325, -3.9233, -4.5767, -4.5845, -4.5902, -4.5936, -4.5968, -4.5894, -4.5939, -2.9242, -2.9264, -2.9269, -2.9347, -2.5343, -3.059, -3.0878, -3.0986, -3.1166, -3.129, -3.1993, -2.9031, -2.9108, -2.9084, -4.2229, -4.243, -4.2481, -4.2558, -4.2606, -4.2019, -4.2133, -4.2216, -4.2294, -4.2322, -4.2416, -4.2424, -4.2519, -4.2551, -4.2561, -4.1907, -4.2082, -4.2096, -4.2024, -2.5892, -2.5912, -2.5921, -2.598, -2.5998, -2.5747, -3.8292, -3.8622, -3.8639, -3.8668, -3.87, -3.8793, -3.8855, -3.9088, -3.9096, -3.9136, -3.9187, -3.9194, -3.9201, -3.9204, -3.9224, -3.9243, -3.927, -3.9317, -3.9339, -3.9353, -3.7955, -3.8717, -3.8736, -3.8855, -3.8862, -3.894, -3.8835], \"loglift\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.5457, 0.4825, 0.4818, 0.4788, 0.3529, 0.3523, 0.3514, 0.3512, 0.3489, 0.3482, 0.3473, 0.3453, 0.3406, 0.3372, -0.0698, -0.0781, -0.2528, -0.3756, -0.4757, -0.5022, -0.5333, -0.5595, -0.6133, -0.7282, -0.7617, -0.7675, -0.7758, -0.7769, -0.7792, -0.9705, -0.9767, 0.6928, 0.691, 0.6903, 0.6842, 0.6608, 0.5602, 0.5324, 0.5211, 0.5037, 0.4909, 0.4232, 0.2881, 0.2803, 0.0768, -0.4085, -0.4281, -0.4314, -0.4382, -0.4436, -0.5862, -0.5976, -0.6068, -0.6147, -0.617, -0.6261, -0.6277, -0.6363, -0.6397, -0.6405, -1.0009, -1.0187, -1.0197, -1.3112, 1.2284, 1.2258, 1.2246, 1.2169, 1.2146, 0.4104, -0.2066, -0.2419, -0.2438, -0.2469, -0.2502, -0.2602, -0.2666, -0.2915, -0.2924, -0.2966, -0.303, -0.3038, -0.3045, -0.3048, -0.307, -0.309, -0.3118, -0.3168, -0.3192, -0.3207, -0.6003, -0.6806, -0.6826, -0.6956, -0.6963, -0.7045, -0.9924]}, \"token.table\": {\"Topic\": [1, 2, 1, 1, 1, 1, 1, 2, 3, 1, 2, 3, 2, 1, 2, 3, 2, 1, 1, 2, 2, 1, 2, 3, 2, 2, 2, 3, 1, 3, 1, 1, 2, 1, 2, 1, 1], \"Freq\": [0.8637372750018115, 0.8659803578149075, 0.5647760676854497, 0.8643849604889509, 0.5647891780456282, 0.8638697904300582, 0.8636628459907907, 0.8705152726255397, 1.0573630446724822, 0.4602059950613118, 0.4602059950613118, 0.4602059950613118, 0.8680925348495789, 0.5645775496538465, 0.8685452049673958, 1.0552009527650315, 0.8673351712921569, 0.8378886966738939, 0.8642738136862269, 0.8675699025063551, 0.8682541410554505, 0.5654948217808083, 0.5654948217808083, 1.0546295062109312, 0.8657057930990445, 0.8659340209745227, 0.8684667647050559, 1.0579995714140842, 0.8644975923672755, 1.057082864519784, 0.8645111594590058, 0.864565030503443, 0.5677423844469763, 0.5654428198061641, 0.5654428198061641, 0.8641295118903841, 0.8646116935798562], \"Term\": [\"abstract\", \"algorithm\", \"artificial\", \"collection\", \"deep\", \"discovering\", \"documents\", \"extracting\", \"fascinating\", \"field\", \"field\", \"field\", \"information\", \"intelligence\", \"involves\", \"language\", \"lda\", \"learning\", \"machine\", \"meaningful\", \"mining\", \"modeling\", \"modeling\", \"natural\", \"nlp\", \"popular\", \"process\", \"processing\", \"revolutionized\", \"study\", \"subsets\", \"technique\", \"text\", \"topic\", \"topic\", \"topics\", \"within\"]}, \"R\": 30, \"lambda.step\": 0.01, \"plot.opts\": {\"xlab\": \"PC1\", \"ylab\": \"PC2\"}, \"topic.order\": [2, 3, 1]};\n",
       "\n",
       "function LDAvis_load_lib(url, callback){\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = callback;\n",
       "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "}\n",
       "\n",
       "if(typeof(LDAvis) !== \"undefined\"){\n",
       "   // already loaded: just create the visualization\n",
       "   !function(LDAvis){\n",
       "       new LDAvis(\"#\" + \"ldavis_el16322205661301664798383600\", ldavis_el16322205661301664798383600_data);\n",
       "   }(LDAvis);\n",
       "}else if(typeof define === \"function\" && define.amd){\n",
       "   // require.js is available: use it to load d3/LDAvis\n",
       "   require.config({paths: {d3: \"https://d3js.org/d3.v5\"}});\n",
       "   require([\"d3\"], function(d3){\n",
       "      window.d3 = d3;\n",
       "      LDAvis_load_lib(\"https://cdn.jsdelivr.net/gh/bmabey/pyLDAvis@3.4.0/pyLDAvis/js/ldavis.v3.0.0.js\", function(){\n",
       "        new LDAvis(\"#\" + \"ldavis_el16322205661301664798383600\", ldavis_el16322205661301664798383600_data);\n",
       "      });\n",
       "    });\n",
       "}else{\n",
       "    // require.js not available: dynamically load d3 & LDAvis\n",
       "    LDAvis_load_lib(\"https://d3js.org/d3.v5.js\", function(){\n",
       "         LDAvis_load_lib(\"https://cdn.jsdelivr.net/gh/bmabey/pyLDAvis@3.4.0/pyLDAvis/js/ldavis.v3.0.0.js\", function(){\n",
       "                 new LDAvis(\"#\" + \"ldavis_el16322205661301664798383600\", ldavis_el16322205661301664798383600_data);\n",
       "            })\n",
       "         });\n",
       "}\n",
       "</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Display the visualization in the notebook\n",
    "pyLDAvis.display(vis_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98af9413-b4cb-4061-be3b-989a83608649",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}