French vs English vocabulary in Mulan.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    English stats:\n",
      "    total words : 5963\n",
      "    unique words : 1416\n",
      "    top 10 : ['the', 'you', 'i', 'to', 'a', 'your', 'and', 'my', 'of', 'me']\n",
      "    bottom 10 : ['volunteering', 'worried,', 'beads', 'fall', 'proclamation', 'begging', 'chorus,', 'temple', 'lost', 'forget']\n",
      "    \n",
      "\n",
      "    French stats:\n",
      "    total words : 4669\n",
      "    unique words : 1554\n",
      "    top 10 : ['de', 'je', 'la', 'tu', 'le', 'un', 'les', 'pas', 'et', 'à']\n",
      "    bottom 10 : [\"l'allumes\", 'fera', 'regarde,', 'temple', \"n'ont\", 'croira', 'risque,', 'epaules', 'faite', 'descends']\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "eng = open(\"./data/mulan_eng.dtc\").read().lower()\n",
    "fr = open(\"./data/mulan_fr.dtc\").read().lower()\n",
    "import re\n",
    "def words(subtitles):\n",
    "    stripped = re.sub(\"{.*}\", \"\", subtitles, flags = re.UNICODE)\n",
    "    return re.findall(r\"[^\\|?!\\s\\.-]+\", stripped, flags = re.UNICODE)\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "eng_words = Counter(words(eng))\n",
    "fr_words = Counter(words(fr))\n",
    "def rank(word_list):\n",
    "    return sorted(word_list.keys(), key = word_list.get)[::-1]\n",
    "\n",
    "def stats(word_list, name):\n",
    "    ranked = rank(word_list)\n",
    "    stats_dict = {\n",
    "        \"name\" : name,\n",
    "        \"total\" : sum(word_list.values()),\n",
    "        \"unique\" : len(word_list.keys()),\n",
    "        \"top 10\" : ranked[:10],\n",
    "        \"bottom 10\" : ranked[-10:],\n",
    "    }\n",
    "    \n",
    "    return \"\"\"\n",
    "    {name} stats:\n",
    "    total words : {total}\n",
    "    unique words : {unique}\n",
    "    top 10 : {top 10}\n",
    "    bottom 10 : {bottom 10}\n",
    "    \"\"\".format(**stats_dict)\n",
    "\n",
    "print(stats(eng_words, \"English\"))\n",
    "print(stats(fr_words, \"French\"))\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}