generate_htmls.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import sys\n",
    "\n",
    "sys.path.insert(0, sys.path[0] + '/scripts')\n",
    "from bert import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mismatchable_params = set(['schedule_lin', 'schedule_cos', 'schedule_exp', 'schedule_none'])\n",
    "\n",
    "def get_type(line):\n",
    "    '''Determine the line type (function, class, or other/None)\n",
    "        line: string of a line in python script\n",
    "    '''\n",
    "    if line[:3] == 'def':\n",
    "        return 'def'\n",
    "    if line[:5] == 'class': \n",
    "        return 'class'\n",
    "    return None\n",
    "\n",
    "def script2doc(f_name):\n",
    "    '''Parse single script to doc.\n",
    "        f_name: file path for script\n",
    "    '''\n",
    "    def process_function():\n",
    "        '''Parse function.'''\n",
    "        nonlocal start, docs\n",
    "        name = f[start].split('(', 1)[0].split()[-1]\n",
    "        # while loop to find start of comment\n",
    "        end = start\n",
    "        while end < len(f):\n",
    "            if f[end].strip().startswith(\"'''\"):\n",
    "                break\n",
    "            end += 1\n",
    "        # grab comment and params\n",
    "        comment = f[end][:-1].strip().replace(\"'''\", '')\n",
    "        params = (''.join(f[start:end])).split('(', 1)[1].rsplit(')', 1)[0].replace(' ', '').replace('\\n', '').split(',')\n",
    "        if len(params) == 1 and not params[0]:\n",
    "            params = []\n",
    "            \n",
    "        doc = list(map(lambda l: l.strip(), globals()[name].__doc__.split('\\n')))\n",
    "        comment = doc[0]\n",
    "        com_params = {}\n",
    "        if len(doc) > 1:\n",
    "            for p in doc[1:-1]:\n",
    "                k, v = p.split(':', 1)\n",
    "                com_params[k] = v[1:] # whitespace after colon\n",
    "        docs.append(('DEF', name, params, comment, com_params))\n",
    "        \n",
    "        assert comment[-1] == '.', print(name)\n",
    "        if len(params) != len(com_params) and name not in mismatchable_params: \n",
    "            print(name, '\\n', params, '\\n', com_params, '\\n')\n",
    "        \n",
    "    def process_class():\n",
    "        '''Parse class.'''\n",
    "        nonlocal start, docs\n",
    "        name = f[start].split()[1].rsplit(')')[0] + ')'\n",
    "        \n",
    "        end, start = start+1, -1\n",
    "        while end < len(f):\n",
    "            if '__init__' in f[end] and start == -1:\n",
    "                start = end\n",
    "            if ')' in f[end]:\n",
    "                break\n",
    "            end += 1\n",
    "        assert start != -1, name\n",
    "        \n",
    "        params = ''.join(f[start: end+1])\n",
    "        params = params.split('(', 1)[1].rsplit(')', 1)[0]\n",
    "        params = params.replace(' ', '').replace('\\n', '')\n",
    "        params = list(params)\n",
    "        \n",
    "        in_bracket, stored_commas = False, []\n",
    "        for i, c in enumerate(params):\n",
    "            if c == '(':\n",
    "                in_bracket = True\n",
    "            elif c == ')':\n",
    "                in_bracket = False\n",
    "            elif c == ',' and in_bracket:\n",
    "                params[i] = '$' # temporarily replace , with $\n",
    "        params = ''.join(params).split(',')[1:]\n",
    "        for i in range(len(params)): \n",
    "            params[i] = params[i].replace('$', ',')\n",
    "        start = end + 1\n",
    "        \n",
    "        doc = list(map(lambda l: l.strip(), globals()[name.split('(', 1)[0]].__init__.__doc__.split('\\n')))\n",
    "        comment = doc[0]\n",
    "        com_params = {}\n",
    "        if len(doc) > 1:\n",
    "            for p in doc[1:-1]:\n",
    "                k, v = p.split(':', 1)\n",
    "                com_params[k] = v[1:]\n",
    "        docs.append(('CLS', name, params, comment, com_params))\n",
    "        \n",
    "        assert comment[-1] == '.', print(name)\n",
    "        if len(params) != len(com_params): print(name, '\\n', params, '\\n', com_params, '\\n')\n",
    "                \n",
    "    docs = []\n",
    "    with open(f_name, 'r') as f:\n",
    "        f = list(f)\n",
    "        start = 0\n",
    "        while start < len(f):\n",
    "            type_ = get_type(f[start])\n",
    "            if type_ == 'def':\n",
    "                process_function()\n",
    "            elif type_ == 'class':\n",
    "                process_class()\n",
    "            start += 1\n",
    "    return docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean():\n",
    "    '''Purge generated html directory.'''\n",
    "    files = glob.glob('docs/assets/generated_htmls/*')\n",
    "    for f_name in files:\n",
    "        if f_name.endswith('txt'):\n",
    "            os.remove(f_name)\n",
    "\n",
    "def scripts2docs():\n",
    "    '''Parse all scripts and return a mapping from scripts to documentations.'''\n",
    "    all_docs = {}\n",
    "#     all_docs['scripts/data_block.py'] = script2doc('scripts/data_block.py')\n",
    "#     return all_docs\n",
    "    for name in glob.glob('scripts/*'):\n",
    "        if name.endswith('py'):\n",
    "            all_docs[name] = script2doc(name)\n",
    "    return all_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "template1 = '''\n",
    "                                <div class=\"section-block-small\">\n",
    "                                    <div class=\"doc-block\">\n",
    "                                        <table>\n",
    "                                           <td><span class=\"label\">&nbsp;{}</span></td>\n",
    "                                            <td><span class=\"name\">{}</span></td>\n",
    "                                            <td class=\"expand\"><span class=\"params\">({})</span></td>\n",
    "                                        </table>\n",
    "                                        <p><span class=\"desc\">{}</span></p>\n",
    "                                    </div>\n",
    "                                </div>'''\n",
    "\n",
    "\n",
    "template2_head = '''                                <div class=\"section-block-small\">\n",
    "                                    <div class=\"doc-block\">\n",
    "                                        <table>\n",
    "                                           <td><span class=\"label\">&nbsp;{}</span></td>\n",
    "                                            <td><span class=\"name\">{}</span></td>\n",
    "                                            <td class=\"expand\"><span class=\"params\">({})</span></td>\n",
    "                                        </table>\n",
    "                                        <p><span class=\"desc\">{}</span></p>\n",
    "                                        <ul class=\"list\">'''\n",
    "\n",
    "template2_tail = '''\n",
    "                                        </ul>\n",
    "                                    </div>\n",
    "                                </div>'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def doc2html(doc):\n",
    "    '''Convert single doc to html.\n",
    "        doc: documentation pulled from python script\n",
    "    '''\n",
    "    html = []\n",
    "    for type_, name, params, comment, com_params in doc:\n",
    "        if not com_params:\n",
    "            html.append(template1.format(type_, name, ', '.join(params), comment))\n",
    "        else:\n",
    "            cur = template2_head.format(type_, name, ', '.join(params), comment)\n",
    "            for k, v in com_params.items():\n",
    "                cur += f\"\\n                                           <li><b>{k}</b>: {v}</li>\"\n",
    "            cur += template2_tail\n",
    "            html.append(cur)\n",
    "    return '\\n'.join(html)\n",
    "\n",
    "def docs2htmls(all_docs):\n",
    "    '''Convert all docs to a map from file name to html files.\n",
    "        all_docs: a map of script names to documentations\n",
    "    '''\n",
    "    htmls = {}\n",
    "    for name, doc in all_docs.items():\n",
    "        htmls[name] = doc2html(doc)\n",
    "    return htmls\n",
    "\n",
    "def store_htmls(htmls):\n",
    "    '''Store htmls to docs/assets/generated_htmls.\n",
    "        htmls: a map of script names to html files\n",
    "    '''\n",
    "    for name, html in htmls.items():\n",
    "        name = 'docs/assets/generated_htmls/' + name.split('/')[1][:-3] + '.html'\n",
    "        with open(name, 'w') as f:\n",
    "            f.write(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scripts2htmls():\n",
    "    '''Wrapper function for going from python scripts to html'''\n",
    "    all_docs = scripts2docs()\n",
    "    htmls = docs2htmls(all_docs)\n",
    "    store_htmls(htmls)\n",
    "    return htmls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean()\n",
    "htmls = scripts2htmls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}