ipynb file to extract wiki articles generated in google colab

wiki_dump_extractor.ipynb

@@ -0,0 +1,106 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "AIN1eV12yzVv",
+        "outputId": "d8543725-1023-4b42-8392-4434fe32adff"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Collecting wikiextractor\n",
+            "  Downloading wikiextractor-3.0.6-py3-none-any.whl (46 kB)\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.4/46.4 kB\u001b[0m \u001b[31m478.1 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hInstalling collected packages: wikiextractor\n",
+            "Successfully installed wikiextractor-3.0.6\n"
+          ]
+        }
+      ],
+      "source": [
+        "pip install wikiextractor"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!wget https://dumps.wikimedia.org/bnwiki/latest/bnwiki-latest-pages-articles.xml.bz2\n",
+        "!bzip2 -d bnwiki-latest-pages-articles.xml.bz2\n",
+        "!python -m wikiextractor.WikiExtractor bnwiki-latest-pages-articles.xml"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "G1HIxA3u17o0",
+        "outputId": "c2997dd8-f849-4be6-83c1-0862f41b0fee"
+      },
+      "execution_count": 31,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "--2024-05-10 09:31:22--  https://dumps.wikimedia.org/bnwiki/latest/bnwiki-latest-pages-articles.xml.bz2\n",
+            "Resolving dumps.wikimedia.org (dumps.wikimedia.org)... 208.80.154.71, 2620:0:861:3:208:80:154:71\n",
+            "Connecting to dumps.wikimedia.org (dumps.wikimedia.org)|208.80.154.71|:443... connected.\n",
+            "HTTP request sent, awaiting response... 200 OK\n",
+            "Length: 380404074 (363M) [application/octet-stream]\n",
+            "Saving to: ‘bnwiki-latest-pages-articles.xml.bz2’\n",
+            "\n",
+            "bnwiki-latest-pages 100%[===================>] 362.78M  4.84MB/s    in 77s     \n",
+            "\n",
+            "2024-05-10 09:32:39 (4.71 MB/s) - ‘bnwiki-latest-pages-articles.xml.bz2’ saved [380404074/380404074]\n",
+            "\n",
+            "INFO: Preprocessing 'bnwiki-latest-pages-articles.xml' to collect template definitions: this may take some time.\n",
+            "INFO: Preprocessed 100000 pages\n",
+            "INFO: Preprocessed 200000 pages\n",
+            "INFO: Preprocessed 300000 pages\n",
+            "INFO: Preprocessed 400000 pages\n",
+            "INFO: Preprocessed 500000 pages\n",
+            "INFO: Preprocessed 600000 pages\n",
+            "INFO: Preprocessed 700000 pages\n",
+            "INFO: Loaded 166186 templates in 54.8s\n",
+            "INFO: Starting page extraction from bnwiki-latest-pages-articles.xml.\n",
+            "INFO: Using 1 extract processes.\n",
+            "INFO: Extracted 100000 articles (1218.9 art/s)\n",
+            "INFO: Extracted 200000 articles (939.3 art/s)\n",
+            "INFO: Extracted 300000 articles (741.1 art/s)\n",
+            "INFO: Finished 1-process extraction of 383474 articles in 459.8s (833.9 art/s)\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!tar -cjf archive.tar.bz2 text\n"
+      ],
+      "metadata": {
+        "id": "IG0pMSjBzBQI"
+      },
+      "execution_count": 40,
+      "outputs": []
+    }
+  ]
+}