nlp-nepali-training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-16T19:15:59.873070Z",
     "iopub.status.busy": "2024-11-16T19:15:59.872420Z",
     "iopub.status.idle": "2024-11-16T19:16:18.502450Z",
     "shell.execute_reply": "2024-11-16T19:16:18.501644Z",
     "shell.execute_reply.started": "2024-11-16T19:15:59.873036Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "from transformers import AutoModelForCausalLM, GemmaConfig, AutoTokenizer, AutoModel, MistralConfig, MistralModel, MistralForCausalLM, LlamaConfig, LlamaForCausalLM\n",
    "from transformers import Trainer, DataCollatorForLanguageModeling, TrainingArguments, TrainerCallback\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "import json\n",
    "import pickle\n",
    "import pandas as pd\n",
    "from datasets import Dataset\n",
    "import math\n",
    "import wandb\n",
    "import ast"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-16T19:16:18.504651Z",
     "iopub.status.busy": "2024-11-16T19:16:18.504286Z",
     "iopub.status.idle": "2024-11-16T19:16:18.717621Z",
     "shell.execute_reply": "2024-11-16T19:16:18.716824Z",
     "shell.execute_reply.started": "2024-11-16T19:16:18.504615Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "token_path = \"/kaggle/input/sangrah_tokenizer/transformers/default/1/sangrah_tokenizers/sangrah.csv_SentencePieceBPETokenizer_50000_transformer\" ## CHANGE TOKENIZER PATH HERE\n",
    "tokenizer = AutoTokenizer.from_pretrained(token_path)\n",
    "# len(tokenizer.vocab), tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LlamaConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-16T19:16:18.719168Z",
     "iopub.status.busy": "2024-11-16T19:16:18.718809Z",
     "iopub.status.idle": "2024-11-16T19:16:18.754452Z",
     "shell.execute_reply": "2024-11-16T19:16:18.753523Z",
     "shell.execute_reply.started": "2024-11-16T19:16:18.719128Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LlamaConfig {\n",
       "  \"attention_bias\": false,\n",
       "  \"attention_dropout\": 0.0,\n",
       "  \"bos_token_id\": 1,\n",
       "  \"eos_token_id\": 2,\n",
       "  \"head_dim\": 64,\n",
       "  \"hidden_act\": \"silu\",\n",
       "  \"hidden_size\": 512,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 1024,\n",
       "  \"max_position_embeddings\": 2048,\n",
       "  \"mlp_bias\": false,\n",
       "  \"model_type\": \"llama\",\n",
       "  \"num_attention_heads\": 8,\n",
       "  \"num_hidden_layers\": 16,\n",
       "  \"num_key_value_heads\": 2,\n",
       "  \"pretraining_tp\": 1,\n",
       "  \"rms_norm_eps\": 1e-06,\n",
       "  \"rope_scaling\": null,\n",
       "  \"rope_theta\": 10000.0,\n",
       "  \"tie_word_embeddings\": false,\n",
       "  \"transformers_version\": \"4.45.1\",\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 49152\n",
       "}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## CHNAGE MODEL CONFIGURATION HERE\n",
    "config = LlamaConfig(hidden_size=512,\n",
    "                     vocab_size=len(tokenizer.vocab),\n",
    "                     num_attention_heads=8,\n",
    "                     num_key_value_heads=2,\n",
    "                     num_hidden_layers=16,\n",
    "                     intermediate_size=1024)\n",
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-16T19:16:18.756934Z",
     "iopub.status.busy": "2024-11-16T19:16:18.756622Z",
     "iopub.status.idle": "2024-11-16T19:16:20.225356Z",
     "shell.execute_reply": "2024-11-16T19:16:20.224318Z",
     "shell.execute_reply.started": "2024-11-16T19:16:18.756894Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LlamaForCausalLM(\n",
       "  (model): LlamaModel(\n",
       "    (embed_tokens): Embedding(49152, 512)\n",
       "    (layers): ModuleList(\n",
       "      (0-15): 16 x LlamaDecoderLayer(\n",
       "        (self_attn): LlamaSdpaAttention(\n",
       "          (q_proj): Linear(in_features=512, out_features=512, bias=False)\n",
       "          (k_proj): Linear(in_features=512, out_features=128, bias=False)\n",
       "          (v_proj): Linear(in_features=512, out_features=128, bias=False)\n",
       "          (o_proj): Linear(in_features=512, out_features=512, bias=False)\n",
       "          (rotary_emb): LlamaRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): LlamaMLP(\n",
       "          (gate_proj): Linear(in_features=512, out_features=1024, bias=False)\n",
       "          (up_proj): Linear(in_features=512, out_features=1024, bias=False)\n",
       "          (down_proj): Linear(in_features=1024, out_features=512, bias=False)\n",
       "          (act_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): LlamaRMSNorm((512,), eps=1e-06)\n",
       "        (post_attention_layernorm): LlamaRMSNorm((512,), eps=1e-06)\n",
       "      )\n",
       "    )\n",
       "    (norm): LlamaRMSNorm((512,), eps=1e-06)\n",
       "    (rotary_emb): LlamaRotaryEmbedding()\n",
       "  )\n",
       "  (lm_head): Linear(in_features=512, out_features=49152, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## MODEL OVERVIEW\n",
    "model = LlamaForCausalLM(config)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-16T19:16:20.227123Z",
     "iopub.status.busy": "2024-11-16T19:16:20.226506Z",
     "iopub.status.idle": "2024-11-16T19:16:20.233346Z",
     "shell.execute_reply": "2024-11-16T19:16:20.232324Z",
     "shell.execute_reply.started": "2024-11-16T19:16:20.227090Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86.000128\n"
     ]
    }
   ],
   "source": [
    "## TOTAL PARAMS\n",
    "total_param=0\n",
    "for i,j in model.named_parameters():\n",
    "    total_param += j.numel()\n",
    "print(total_param/(10**6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-16T19:16:20.234980Z",
     "iopub.status.busy": "2024-11-16T19:16:20.234670Z",
     "iopub.status.idle": "2024-11-16T19:16:59.095757Z",
     "shell.execute_reply": "2024-11-16T19:16:59.094725Z",
     "shell.execute_reply.started": "2024-11-16T19:16:20.234948Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[5, 20632, 11371, 214, 569, 6748, 438, 11307, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[133, 1778, 405, 5839, 1007, 28641, 454, 235, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[132, 23605, 875, 8153, 566, 4104, 10071, 2003...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[22, 405, 1241, 6, 132, 5, 438, 1830, 224, 181...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[29497, 454, 5991, 599, 7045, 315, 6, 132, 5, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33018</th>\n",
       "      <td>[21248, 33, 158, 1591, 45741, 4591, 487, 75, 9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33019</th>\n",
       "      <td>[189, 41472, 33, 489, 4366, 680, 9128, 31661, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33020</th>\n",
       "      <td>[489, 21046, 250, 9278, 3100, 9278, 459, 815, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33021</th>\n",
       "      <td>[25, 192, 241, 663, 284, 7565, 132, 486, 31275...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33022</th>\n",
       "      <td>[8928, 32077, 30679, 10733, 6515, 7727, 287, 3...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>33023 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               input_ids\n",
       "0      [5, 20632, 11371, 214, 569, 6748, 438, 11307, ...\n",
       "1      [133, 1778, 405, 5839, 1007, 28641, 454, 235, ...\n",
       "2      [132, 23605, 875, 8153, 566, 4104, 10071, 2003...\n",
       "3      [22, 405, 1241, 6, 132, 5, 438, 1830, 224, 181...\n",
       "4      [29497, 454, 5991, 599, 7045, 315, 6, 132, 5, ...\n",
       "...                                                  ...\n",
       "33018  [21248, 33, 158, 1591, 45741, 4591, 487, 75, 9...\n",
       "33019  [189, 41472, 33, 489, 4366, 680, 9128, 31661, ...\n",
       "33020  [489, 21046, 250, 9278, 3100, 9278, 459, 815, ...\n",
       "33021  [25, 192, 241, 663, 284, 7565, 132, 486, 31275...\n",
       "33022  [8928, 32077, 30679, 10733, 6515, 7727, 287, 3...\n",
       "\n",
       "[33023 rows x 1 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['input_ids'],\n",
      "    num_rows: 33023\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "dataset = pd.read_csv(\"/kaggle/input/tokenized-sangrah/kaggle/working/tokenized_data.csv\") ## CHANGE THE PATH TO TOKENIZED DATASET HERE\n",
    "# dataset = dataset.drop(columns = 'Unnamed: 0')                                                      ## REMOVED A COLUM THAT WAS UNNECESSARY\n",
    "dataset['input_ids'] = dataset['input_ids'].apply(ast.literal_eval)                                 ## CONVERTED THE Str(List[Int]) to List[Int] (IMPORTANT)\n",
    "display(dataset)                                                                                    \n",
    "\n",
    "tokenized_data = Dataset.from_pandas(dataset)                                                       ## CONVERTED TO A DATASET OBJECT\n",
    "\n",
    "print(tokenized_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-16T19:16:59.097078Z",
     "iopub.status.busy": "2024-11-16T19:16:59.096799Z",
     "iopub.status.idle": "2024-11-16T19:16:59.130729Z",
     "shell.execute_reply": "2024-11-16T19:16:59.129909Z",
     "shell.execute_reply.started": "2024-11-16T19:16:59.097048Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['input_ids'],\n",
      "    num_rows: 29720\n",
      "}) Dataset({\n",
      "    features: ['input_ids'],\n",
      "    num_rows: 3303\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "train_test_split = tokenized_data.train_test_split(test_size=0.1)\n",
    "train_dataset = train_test_split[\"train\"]\n",
    "eval_dataset = train_test_split[\"test\"]\n",
    "\n",
    "# Data Collator for Language Modeling\n",
    "data_collator = DataCollatorForLanguageModeling(\n",
    "    tokenizer=tokenizer,\n",
    "    mlm=False  # For causal language modeling\n",
    ")\n",
    "print(train_dataset, eval_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-16T19:16:59.132506Z",
     "iopub.status.busy": "2024-11-16T19:16:59.131873Z",
     "iopub.status.idle": "2024-11-16T19:17:10.858452Z",
     "shell.execute_reply": "2024-11-16T19:17:10.857563Z",
     "shell.execute_reply.started": "2024-11-16T19:16:59.132462Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  ········································\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "feea1fefccc3417fbf29b04e3fa4ff6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='Waiting for wandb.init()...\\r'), FloatProgress(value=0.01111411513333375, max=1.0)…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.18.3"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/kaggle/working/wandb/run-20241116_191707-wk7a20js</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/hrriday-ruparel-indian-institute-of-technology-gandhinagar/nlp_nepali_training/runs/wk7a20js' target=\"_blank\">llama_nepali</a></strong> to <a href='https://wandb.ai/hrriday-ruparel-indian-institute-of-technology-gandhinagar/nlp_nepali_training' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/hrriday-ruparel-indian-institute-of-technology-gandhinagar/nlp_nepali_training' target=\"_blank\">https://wandb.ai/hrriday-ruparel-indian-institute-of-technology-gandhinagar/nlp_nepali_training</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/hrriday-ruparel-indian-institute-of-technology-gandhinagar/nlp_nepali_training/runs/wk7a20js' target=\"_blank\">https://wandb.ai/hrriday-ruparel-indian-institute-of-technology-gandhinagar/nlp_nepali_training/runs/wk7a20js</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<button onClick=\"this.nextSibling.style.display='block';this.style.display='none';\">Display W&B run</button><iframe src='https://wandb.ai/hrriday-ruparel-indian-institute-of-technology-gandhinagar/nlp_nepali_training/runs/wk7a20js?jupyter=true' style='border:none;width:100%;height:420px;display:none;'></iframe>"
      ],
      "text/plain": [
       "<wandb.sdk.wandb_run.Run at 0x7d46bf070340>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## WANDB = Weights and Biases (website) that tracks the training process, stores necesary information like loss and also the trained models. You will have to create an account on Weights and Biases website.\n",
    "## Once account is created, only then run this cell and paste your API key in the text prompt. Your API key is available at https://wandb.ai/authorize or in your home page when you open Weights and Biases account.\n",
    "wandb.init(project=\"nlp_nepali_training\", name=\"sangrah\") ## CHANGE AS PER YOUR WISH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-16T19:17:10.860497Z",
     "iopub.status.busy": "2024-11-16T19:17:10.859820Z",
     "iopub.status.idle": "2024-11-16T19:17:11.249185Z",
     "shell.execute_reply": "2024-11-16T19:17:11.248389Z",
     "shell.execute_reply.started": "2024-11-16T19:17:10.860452Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py:494: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    }
   ],
   "source": [
    "## NO NEED TO CHANGE ANYTHING HERE\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./results\",                # Set a local path to store results (checkpoints)\n",
    "    eval_strategy=\"steps\",           # Evaluate after every epoch\n",
    "    logging_dir=\"./logs\",                  # Directory for logs\n",
    "    logging_steps=50,                      # Log every 50 steps\n",
    "    save_steps=1000,                       # Save checkpoints every 1000 steps\n",
    "    eval_steps = 1000,\n",
    "    save_total_limit=3,                    # Limit the number of saved checkpoints\n",
    "    per_device_train_batch_size=8,         # Adjust batch size based on your memory\n",
    "    per_device_eval_batch_size=8,\n",
    "    num_train_epochs=20,                    # Number of epochs\n",
    "    report_to=\"wandb\",                     # Log to W&B\n",
    "    logging_first_step=True,\n",
    "    load_best_model_at_end=True,           # Load the best model based on evaluation\n",
    "    save_strategy=\"steps\",                 # Save model after every epoch (or steps)\n",
    "    push_to_hub=False,                       # Avoid pushing to hub (unless needed)\n",
    "    fp16 = True\n",
    ")\n",
    "\n",
    "class PerplexityCallback(TrainerCallback):\n",
    "    def __init__(self):\n",
    "        self.perplexity_by_epoch = []\n",
    "\n",
    "    def on_evaluate(self, args, state, control, metrics=None, **kwargs):\n",
    "        if metrics and \"eval_loss\" in metrics:\n",
    "            perplexity = math.exp(metrics[\"eval_loss\"])\n",
    "            self.perplexity_by_epoch.append({\"epoch\": state.epoch, \"perplexity\": perplexity})\n",
    "            wandb.log({\"eval_perplexity\": perplexity})\n",
    "            \n",
    "    def on_log(self, args, state, control, logs=None, **kwargs):\n",
    "        if \"eval_loss\" in logs:\n",
    "            perplexity = math.exp(logs[\"eval_loss\"])\n",
    "            logs[\"eval_perplexity\"] = perplexity\n",
    "            wandb.log({\"eval_perplexity\": perplexity})\n",
    "\n",
    "class ModelSaveCallback(TrainerCallback):\n",
    "    def __init__(self, save_interval=0.25):\n",
    "        super().__init__()\n",
    "        self.save_interval = save_interval  # Interval to save model (fraction of epoch)\n",
    "        self.last_saved_step = 0\n",
    "\n",
    "    def on_step_end(self, args, state, control, **kwargs):\n",
    "        # Calculate current progress in epochs\n",
    "        current_epoch_fraction = state.global_step / state.max_steps * args.num_train_epochs\n",
    "        \n",
    "        # Check if it's time to save the model\n",
    "        if (current_epoch_fraction - self.last_saved_step) >= self.save_interval:\n",
    "            self.last_saved_step = current_epoch_fraction  # Update the last saved step\n",
    "            \n",
    "            # Save the model\n",
    "            model_save_path = f\"{wandb.run.dir}/model_epoch_{current_epoch_fraction:.2f}\"\n",
    "            model.save_pretrained(model_save_path)\n",
    "            \n",
    "            # Upload the saved model to W&B\n",
    "            wandb.save(f\"{model_save_path}/*\")  # Save all files in the directory to W&B\n",
    "            print(f\"Model saved at {current_epoch_fraction:.2f} epochs\")\n",
    "\n",
    "perplexity_callback = PerplexityCallback()\n",
    "\n",
    "# Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=eval_dataset,\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    "    callbacks=[PerplexityCallback(), ModelSaveCallback(save_interval=0.2)] \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-16T19:17:11.251703Z",
     "iopub.status.busy": "2024-11-16T19:17:11.251440Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='8122' max='18580' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [ 8122/18580 1:55:51 < 2:29:13, 1.17 it/s, Epoch 4.37/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Perplexity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>7.331900</td>\n",
       "      <td>7.272457</td>\n",
       "      <td>1440.084574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>6.678000</td>\n",
       "      <td>6.688414</td>\n",
       "      <td>803.047686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>6.340800</td>\n",
       "      <td>6.323770</td>\n",
       "      <td>557.671216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4000</td>\n",
       "      <td>6.092100</td>\n",
       "      <td>6.065169</td>\n",
       "      <td>430.595586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5000</td>\n",
       "      <td>5.853800</td>\n",
       "      <td>5.867444</td>\n",
       "      <td>353.344690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6000</td>\n",
       "      <td>5.569400</td>\n",
       "      <td>5.713341</td>\n",
       "      <td>302.881378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7000</td>\n",
       "      <td>5.486000</td>\n",
       "      <td>5.590077</td>\n",
       "      <td>267.756216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8000</td>\n",
       "      <td>5.373100</td>\n",
       "      <td>5.495382</td>\n",
       "      <td>243.564509</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Saving files without folders. If you want to preserve subdirectories pass base_path to wandb.save, i.e. wandb.save(\"/mnt/folder/file.h5\", base_path=\"/mnt\")\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n",
      "  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    }
   ],
   "source": [
    "wandb.watch(model, log=\"all\", log_freq=100)\n",
    "trainer.train()\n",
    "wandb.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "nvidiaTeslaT4",
   "dataSources": [
    {
     "datasetId": 6101205,
     "sourceId": 9926368,
     "sourceType": "datasetVersion"
    },
    {
     "isSourceIdPinned": true,
     "modelId": 166260,
     "modelInstanceId": 143672,
     "sourceId": 168880,
     "sourceType": "modelInstanceVersion"
    }
   ],
   "dockerImageVersionId": 30787,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}