ViHSD_monolingual_Transformers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ViHSD monolingual Transformers.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c292c0de530540f59ebc5b32da9075c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_436ab91b4a874371897ab36ddff4a866",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bc1bde8fbc4e4139a551de67b3484619",
              "IPY_MODEL_6d51914ec1924e14b4f1ba41b8aec305"
            ]
          }
        },
        "436ab91b4a874371897ab36ddff4a866": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bc1bde8fbc4e4139a551de67b3484619": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_974c50656ee04d3bb7e88ab08e2f1117",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 557,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 557,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_65653957e0fb4ca7bbd151bc3d742db3"
          }
        },
        "6d51914ec1924e14b4f1ba41b8aec305": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3edae93c483f49d695e8d02f085dc52e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 557/557 [00:14&lt;00:00, 37.6B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2abfa7d3c1954bc9b14ff7b8e0f55c63"
          }
        },
        "974c50656ee04d3bb7e88ab08e2f1117": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "65653957e0fb4ca7bbd151bc3d742db3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3edae93c483f49d695e8d02f085dc52e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2abfa7d3c1954bc9b14ff7b8e0f55c63": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d5123c762ed84216b3d539a545fe4d7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_39a31c9caa24490591955df329c0835d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_12497caa12664717a02681ff7ec0dab8",
              "IPY_MODEL_25e3174048ac4365a6518d6f38cf0c0e"
            ]
          }
        },
        "39a31c9caa24490591955df329c0835d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "12497caa12664717a02681ff7ec0dab8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8800e1e25fea4067af8e505bff52c220",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 542923308,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 542923308,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d1df844d8ac34efe982b78399d7e4353"
          }
        },
        "25e3174048ac4365a6518d6f38cf0c0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9e3ebd323b8645298c66fdb5072f8e85",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 543M/543M [00:09&lt;00:00, 59.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1abb70ad749a4ab390088fff542ff025"
          }
        },
        "8800e1e25fea4067af8e505bff52c220": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d1df844d8ac34efe982b78399d7e4353": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9e3ebd323b8645298c66fdb5072f8e85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1abb70ad749a4ab390088fff542ff025": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "41f704cf398f4c9fbb68301c69ff572b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_169640d62f7a401ab90b5dd33b4db655",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f37c71368c0f455b8e2cec0bea795e47",
              "IPY_MODEL_3862244554304c7b884c1b6f44c9b69e"
            ]
          }
        },
        "169640d62f7a401ab90b5dd33b4db655": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f37c71368c0f455b8e2cec0bea795e47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b8104cca1d954f89b39ae33b30675851",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 895321,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 895321,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a6d9a9845c014859a185a48d79df6932"
          }
        },
        "3862244554304c7b884c1b6f44c9b69e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ef04af650ced42e88dab75b34cbc7686",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 895k/895k [00:00&lt;00:00, 1.07MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4144298d9cc6446e9fc1b936250d98ca"
          }
        },
        "b8104cca1d954f89b39ae33b30675851": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a6d9a9845c014859a185a48d79df6932": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ef04af650ced42e88dab75b34cbc7686": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4144298d9cc6446e9fc1b936250d98ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "381dc53fbf434e529ab0b85b28e77a16": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e5a69170d99643d6bd2d6c425a503b58",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2de5e5f7c09e44ada89883a1f37769ab",
              "IPY_MODEL_89e270ab52264cc0ac9a6af6a3d0d89c"
            ]
          }
        },
        "e5a69170d99643d6bd2d6c425a503b58": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2de5e5f7c09e44ada89883a1f37769ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ea0d3ca1af5245cead93c47555e2dd01",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1135173,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1135173,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b8c64234bece4485abfcbc537d268c59"
          }
        },
        "89e270ab52264cc0ac9a6af6a3d0d89c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a6edb6ab5eb74b829f4572d79e70040a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.14M/1.14M [00:00&lt;00:00, 2.38MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_08f3761ed7e5412ba1b41d40c8f4b32f"
          }
        },
        "ea0d3ca1af5245cead93c47555e2dd01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b8c64234bece4485abfcbc537d268c59": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a6edb6ab5eb74b829f4572d79e70040a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "08f3761ed7e5412ba1b41d40c8f4b32f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8YGYUHs7c8JK"
      },
      "source": [
        "#Library"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0kK-Qk5AOg2A",
        "outputId": "4073a96a-86f7-4683-eec0-c291d73b1f0e"
      },
      "source": [
        "pip install transformers==4.3.0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers==4.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5e/bb/61bf4a221150aff2756935d032a83f2e68976b050e8606d990a0f3054179/transformers-4.3.0-py3-none-any.whl (1.8MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8MB 8.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.3.0) (2019.12.20)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 54.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers==4.3.0) (3.10.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 56.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.3.0) (4.41.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==4.3.0) (20.9)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.3.0) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.3.0) (1.19.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.3.0) (3.0.12)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.3.0) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.3.0) (3.4.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.3.0) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.3.0) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.3.0) (1.0.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==4.3.0) (2.4.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.3.0) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.3.0) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.3.0) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.3.0) (3.0.4)\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.45 tokenizers-0.10.2 transformers-4.3.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pIA54XX8QlQ8",
        "outputId": "069eb505-c6e7-40b5-9dbd-7582728b84a9"
      },
      "source": [
        "pip install vncorenlp"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting vncorenlp\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/c2/96a60cf75421ecc740829fa920c617b3dd7fa6791e17554e7c6f3e7d7fca/vncorenlp-1.0.3.tar.gz (2.6MB)\n",
            "\u001b[K     |████████████████████████████████| 2.7MB 7.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from vncorenlp) (2.23.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->vncorenlp) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->vncorenlp) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->vncorenlp) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->vncorenlp) (2.10)\n",
            "Building wheels for collected packages: vncorenlp\n",
            "  Building wheel for vncorenlp (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for vncorenlp: filename=vncorenlp-1.0.3-cp37-none-any.whl size=2645936 sha256=1c3d8db44663b44f477a71ba5a6c36335cb088b6e4c24ef1ddcae3d755060d6c\n",
            "  Stored in directory: /root/.cache/pip/wheels/09/54/8b/043667de6091d06a381d7745f44174504a9a4a56ecc9380c54\n",
            "Successfully built vncorenlp\n",
            "Installing collected packages: vncorenlp\n",
            "Successfully installed vncorenlp-1.0.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cRKP8KTDomE4",
        "outputId": "72b7b6b3-0090-4bd4-ab8d-b3d5d55b1c81"
      },
      "source": [
        "!mkdir -p vncorenlp/models/wordsegmenter\n",
        "!wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/VnCoreNLP-1.1.1.jar\n",
        "!wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/wordsegmenter/vi-vocab\n",
        "!wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/wordsegmenter/wordsegmenter.rdr\n",
        "!mv VnCoreNLP-1.1.1.jar vncorenlp/\n",
        "!mv vi-vocab vncorenlp/models/wordsegmenter/\n",
        "!mv wordsegmenter.rdr vncorenlp/models/wordsegmenter/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-05-09 08:04:26--  https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/VnCoreNLP-1.1.1.jar\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 27412575 (26M) [application/octet-stream]\n",
            "Saving to: ‘VnCoreNLP-1.1.1.jar’\n",
            "\n",
            "VnCoreNLP-1.1.1.jar 100%[===================>]  26.14M  58.4MB/s    in 0.4s    \n",
            "\n",
            "2021-05-09 08:04:27 (58.4 MB/s) - ‘VnCoreNLP-1.1.1.jar’ saved [27412575/27412575]\n",
            "\n",
            "--2021-05-09 08:04:27--  https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/wordsegmenter/vi-vocab\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.108.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 526544 (514K) [application/octet-stream]\n",
            "Saving to: ‘vi-vocab’\n",
            "\n",
            "vi-vocab            100%[===================>] 514.20K  --.-KB/s    in 0.04s   \n",
            "\n",
            "2021-05-09 08:04:27 (12.2 MB/s) - ‘vi-vocab’ saved [526544/526544]\n",
            "\n",
            "--2021-05-09 08:04:27--  https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/wordsegmenter/wordsegmenter.rdr\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.109.133, 185.199.108.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 128508 (125K) [text/plain]\n",
            "Saving to: ‘wordsegmenter.rdr’\n",
            "\n",
            "wordsegmenter.rdr   100%[===================>] 125.50K  --.-KB/s    in 0.01s   \n",
            "\n",
            "2021-05-09 08:04:27 (8.94 MB/s) - ‘wordsegmenter.rdr’ saved [128508/128508]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WrrLkesTdAf1"
      },
      "source": [
        "# Data loader and pre-processing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KM2YfcbjPOwy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ca401dc8-4edd-481d-ffbd-ac3ec796ff01"
      },
      "source": [
        "import pandas as pd\n",
        "# load data\n",
        "train = pd.read_csv(\"drive/My Drive/CODE/HSD/dataset/train.csv\", index_col=False)\n",
        "dev = pd.read_csv(\"drive/My Drive/CODE/HSD/dataset/dev.csv\", index_col=False)\n",
        "test = pd.read_csv(\"drive/My Drive/CODE/HSD/dataset/test.csv\", index_col=False)\n",
        "\n",
        "train.reset_index(drop=True)\n",
        "\n",
        "print(len(train))\n",
        "print(len(dev))\n",
        "print(len(test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "24048\n",
            "2672\n",
            "6680\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Yd9d-GMR46v"
      },
      "source": [
        "from vncorenlp import VnCoreNLP\n",
        "\n",
        "vncorenlp = VnCoreNLP(\"vncorenlp/VnCoreNLP-1.1.1.jar\", annotators=\"wseg\", max_heap_size='-Xmx500m') "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "juq42xDsPjyv"
      },
      "source": [
        "#pre-process\n",
        "import re\n",
        "import numpy as np\n",
        "\n",
        "STOPWORDS = 'drive/My Drive/CODE/HSD/vietnamese-stopwords.txt'\n",
        "with open(STOPWORDS, \"r\") as ins:\n",
        "    stopwords = []\n",
        "    for line in ins:\n",
        "        dd = line.strip('\\n')\n",
        "        stopwords.append(dd)\n",
        "    stopwords = set(stopwords)\n",
        "\n",
        "def filter_stop_words(train_sentences, stop_words):\n",
        "    new_sent = [word for word in train_sentences.split() if word not in stop_words]\n",
        "    train_sentences = ' '.join(new_sent)\n",
        "        \n",
        "    return train_sentences\n",
        "\n",
        "def deEmojify(text):\n",
        "    regrex_pattern = re.compile(pattern = \"[\"\n",
        "        u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "        u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "        u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "        u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "                           \"]+\", flags = re.UNICODE)\n",
        "    return regrex_pattern.sub(r'',text)\n",
        "\n",
        "def preprocess(text, tokenized=True, lowercased=True):\n",
        "    # text = ViTokenizer.tokenize(text)\n",
        "    # text = ' '.join(vncorenlp.tokenize(text)[0])\n",
        "    text = filter_stop_words(text, stopwords)\n",
        "    text = deEmojify(text)\n",
        "    text = text.lower() if lowercased else text\n",
        "    if tokenized:\n",
        "        pre_text = \"\"\n",
        "        sentences = vncorenlp.tokenize(text)\n",
        "        for sentence in sentences:\n",
        "            pre_text += \" \".join(sentence)\n",
        "        text = pre_text\n",
        "    return text\n",
        "\n",
        "def pre_process_features(X, y, tokenized=True, lowercased=True):\n",
        "    X = [preprocess(str(p), tokenized=tokenized, lowercased=lowercased) for p in list(X)]\n",
        "    for idx, ele in enumerate(X):\n",
        "        if not ele:\n",
        "            np.delete(X, idx)\n",
        "            np.delete(y, idx)\n",
        "    return X, y\n",
        "\n",
        "X_train = train['free_text']\n",
        "y_train = train['label_id'].values\n",
        "\n",
        "X_dev = dev['free_text']\n",
        "y_dev = dev['label_id'].values\n",
        "\n",
        "X_test = test['free_text']\n",
        "y_test = test['label_id'].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pD0rg7aFdEC-"
      },
      "source": [
        "# Model\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-cT88Ks1jAJz"
      },
      "source": [
        "## PhoBERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z0Zu0H2EPpYN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 973,
          "referenced_widgets": [
            "c292c0de530540f59ebc5b32da9075c9",
            "436ab91b4a874371897ab36ddff4a866",
            "bc1bde8fbc4e4139a551de67b3484619",
            "6d51914ec1924e14b4f1ba41b8aec305",
            "974c50656ee04d3bb7e88ab08e2f1117",
            "65653957e0fb4ca7bbd151bc3d742db3",
            "3edae93c483f49d695e8d02f085dc52e",
            "2abfa7d3c1954bc9b14ff7b8e0f55c63",
            "d5123c762ed84216b3d539a545fe4d7a",
            "39a31c9caa24490591955df329c0835d",
            "12497caa12664717a02681ff7ec0dab8",
            "25e3174048ac4365a6518d6f38cf0c0e",
            "8800e1e25fea4067af8e505bff52c220",
            "d1df844d8ac34efe982b78399d7e4353",
            "9e3ebd323b8645298c66fdb5072f8e85",
            "1abb70ad749a4ab390088fff542ff025",
            "41f704cf398f4c9fbb68301c69ff572b",
            "169640d62f7a401ab90b5dd33b4db655",
            "f37c71368c0f455b8e2cec0bea795e47",
            "3862244554304c7b884c1b6f44c9b69e",
            "b8104cca1d954f89b39ae33b30675851",
            "a6d9a9845c014859a185a48d79df6932",
            "ef04af650ced42e88dab75b34cbc7686",
            "4144298d9cc6446e9fc1b936250d98ca",
            "381dc53fbf434e529ab0b85b28e77a16",
            "e5a69170d99643d6bd2d6c425a503b58",
            "2de5e5f7c09e44ada89883a1f37769ab",
            "89e270ab52264cc0ac9a6af6a3d0d89c",
            "ea0d3ca1af5245cead93c47555e2dd01",
            "b8c64234bece4485abfcbc537d268c59",
            "a6edb6ab5eb74b829f4572d79e70040a",
            "08f3761ed7e5412ba1b41d40c8f4b32f"
          ]
        },
        "outputId": "cd038d5d-baa5-4b49-bfcc-0f64f4763c6a"
      },
      "source": [
        "# PhoBERT\n",
        "from sklearn.metrics import f1_score, confusion_matrix, accuracy_score, recall_score\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments, BertTokenizer, BertForSequenceClassification\n",
        "\n",
        "import seaborn as sn\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "train_X, train_y = pre_process_features(X_train, y_train, tokenized=True, lowercased = False)\n",
        "dev_X, dev_y = pre_process_features(X_dev, y_dev, tokenized=True, lowercased = False)\n",
        "test_X, test_y = pre_process_features(X_test, y_test, tokenized=True, lowercased = False)\n",
        "  \n",
        "model = AutoModelForSequenceClassification.from_pretrained(\"vinai/phobert-base\", num_labels = 3)\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"vinai/phobert-base\",use_fast=False)\n",
        "\n",
        "import torch\n",
        "\n",
        "class BuildDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "train_encodings = tokenizer(train_X, truncation=True, padding=True, max_length=100)\n",
        "dev_encodings = tokenizer(dev_X, truncation=True, padding=True, max_length=100)\n",
        "test_encodings = tokenizer(test_X, truncation=True, padding=True, max_length=100)\n",
        "\n",
        "train_dataset = BuildDataset(train_encodings, train_y)\n",
        "dev_dataset = BuildDataset(dev_encodings, dev_y)\n",
        "test_dataset = BuildDataset(test_encodings, test_y)\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='drive/MyDrive/CODE/HSD/transformer_model/phobert-v3/',          \n",
        "    num_train_epochs=2,              \n",
        "    per_device_train_batch_size=16,  \n",
        "    per_device_eval_batch_size=16,   \n",
        "    warmup_steps=500,                \n",
        "    weight_decay=0.01,\n",
        "    no_cuda=False,\n",
        "    do_eval=True\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,                         \n",
        "    args=training_args,                  \n",
        "    train_dataset=train_dataset,         \n",
        "    eval_dataset=dev_dataset             \n",
        ")\n",
        "\n",
        "trainer.train()\n",
        "trainer.save_model(\"drive/MyDrive/CODE/HSD/transformer_model/phobert-v3/\")\n",
        "\n",
        "y_pred_classify = trainer.predict(test_dataset)\n",
        "\n",
        "y_pred = np.argmax(y_pred_classify.predictions, axis=-1)\n",
        "y_true = test_y\n",
        "\n",
        "cf1 = confusion_matrix(y_true, y_pred)\n",
        "print(cf1)\n",
        "\n",
        "evaluation = f1_score(y_true, y_pred, average='micro')\n",
        "\n",
        "print(\"F1 - micro: \" + str(evaluation))\n",
        "\n",
        "evaluation = f1_score(y_true, y_pred, average='macro')\n",
        "print(\"F1 - macro: \" + str(evaluation))\n",
        "\n",
        "# Show out the confusion matrix\n",
        "df_cm1 = pd.DataFrame(cf1, index = [\"clean\",\"offensive\",\"hate\"],\n",
        "                  columns = [\"clean\",\"offensive\",\"hate\"])\n",
        "plt.clf()\n",
        "sn.heatmap(df_cm1, annot=True, cmap=\"Greys\",fmt='g', cbar=True, annot_kws={\"size\": 30})"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c292c0de530540f59ebc5b32da9075c9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=557.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d5123c762ed84216b3d539a545fe4d7a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=542923308.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at vinai/phobert-base were not used when initializing RobertaForSequenceClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'lm_head.decoder.bias', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at vinai/phobert-base and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "41f704cf398f4c9fbb68301c69ff572b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=895321.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "381dc53fbf434e529ab0b85b28e77a16",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1135173.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embedding are fine-tuned or trained.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='3006' max='3006' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3006/3006 09:51, Epoch 2/2]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.565600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.424800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.417200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.336700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.338400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.303000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='418' max='418' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [418/418 00:21]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "[[5303  114  131]\n",
            " [ 216  140   88]\n",
            " [ 272   66  350]]\n",
            "F1 - micro: 0.8672155688622755\n",
            "F1 - macro: 0.6195764872951951\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7fe9a7cc5990>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWYAAAD4CAYAAADfPUyRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUxfrA8e9LgEAILYBUKQpIl14EAQURRX4CehWuKKCCBcGAXhEExAai4JXrpQiCIqDSRFBRKYJKJ3QIVQlILwECBELK/P7Yk3M3JNlskt1ks3k/z7NPTpkzZ7LZvDtnzswcMcaglFLKd+TJ7gIopZRKSgOzUkr5GA3MSinlYzQwK6WUj9HArJRSPiavt08gItrtw8sSEhKyuwhKeYSIiAfycDvmGGMyfT5v0BqzUkr5GK/XmJVSKit5oNKd7TQwK6X8Sp48Ob8hQAOzUsqvaGBWSikfo00ZSinlYzQwK6WUj9HArJRSPkYDs1JK+ZiAgIDsLkKm5fzbl0op5URE3H65kVeEiOwSke0iEmZtCxGR5SJy0PpZ3NouIvIfETkkIjtFpKFTPr2s9AdFpFda59XArJTyK54MzJZ7jDH1jTGNrfXXgZXGmGrASmsd4AGgmvXqB0y2yhMCvAk0A5oCbyYG89RoYFZK+RUvBOabPQzMtJZnAl2ctn9pHDYAxUSkLHA/sNwYE2mMuQAsBzq6OoEGZqWUX0lPYBaRfiIS5vTqd1N2BlgmIluc9pU2xpy0lk8Bpa3l8sDfTsces7altj1VevNPKeVX0nPzzxgzFZjqIkkrY8xxEbkFWC4i+2463nhjBk2tMSul/IonmzKMMcetn2eARTjaiE9bTRRYP89YyY8DtzodXsHaltr2VGlgVkr5FU8FZhEpJCKFE5eBDsBuYAmQ2LOiF7DYWl4CPGX1zmgOXLKaPH4BOohIceumXwdrW6q0KUMp5Vc8OMCkNLDIyi8v8JUx5mcR2QzME5FngCPAY1b6pcCDwCEgGugDYIyJFJF3gM1WureNMZEufwdjvPuAEX2CiffpE0yUv/DEE0zKlCnjdsw5deqUTw4T1BqzUsqv6JBspZTyMf4wJFsDs1LKr2iNWSmlfIwGZqWU8jEamD1o1apVtG3b1q20ERERVKlSJcV9DRo0oHnz5jRu3Ji6detSqlQpSpYsSd68eblw4QLh4eEsX76cL774gtOnT7tdvubNm9O3b1/atGlD2bJluX79OocPH2bRokVMmTKF8+fPuzy+UqVKtGrViiZNmtCgQQPKli1LyZIlCQ4OJioqij///JM1a9bwxRdfsGvXLrfL5Q3x8fH8+eef7N69mz179rB7927279/P9evXAejfvz8DBgxIMx9jDBEREXY+e/bsITw8nKtXrwLQpUsX3n///UyXd8qUKXz88cf2uqfy9SZPvcd79uxhx44d7N69mwMHDhAZGcnFixeJi4ujaNGi3H777dx111107dqVUqVKpZnfsWPH7PIk/s0uXboEQJMmTZg1a1bmfvEsoIHZBy1dupQyZcqkuK9s2bKULVuWdu3a8cYbb/DKK68wbdq0NPMcP348oaGhSR7yGBQUREhICI0aNeKll17in//8J6tWrUo1j/fff5/u3bunuK9EiRKUKFGCpk2bEhoaypQpUxg4cCDx8fFpls0bQkNDWb58eabzGTt2LF988UXmC+TCX3/9xaRJk7x6Dm/w1Hv83HPPce7cuRT3nT17lrNnz7JhwwamTJnCkCFDePzxx1PNa9asWbz33nuZLlN205t/XtKlSxeX+6Ojo13uT/ww7tixg8OHD3Pp0iXy589P1apV6dKlCw0bNqRw4cJMnTqVuLg4Pv/881TzGjNmDIMHDwbgypUrTJ8+nU2bNhEcHMwjjzxChw4dKFOmDIsXL+buu+9mx44dqeZ19epVwsLC2LJlCwcPHiQyMhJjDOXLl6d9+/Y88MAD5MmThxdffJHg4GB69Upz2lavuLlfdNGiRSlWrBhHjhzJVD6FChWibNmyHDp0KNNlBEeNfOTIkdy4cYOgoKA0Pxe+xFPvMUDx4sW58847qVGjBhUqVCA4OJjY2FiOHj3KypUrCQ8PJzo6mjfffJO8efPyyCOPuFWmAgUKUKlSJfbv35/uMmUnf6gxY4zx6gvH7ExpvlatWmUSuXtMSq9atWqlmeb111+3zxUZGWny58+fYrr69eub+Ph4Y4wxFy5cMHXr1k2W5s0337Tz2rhxY6rnrFq1aqrnSXzdc8895tq1a3Z+TZs2det3TkhI8Ohr0qRJ5sMPPzRLly41R48eNQkJCWbBggWmevXqpnr16mbChAlu5fP111+b9957zyxevNgcOnTIxMfHm/Xr19v5vPbaa5kq55w5c0z16tVN/fr1zSeffOKxfLPi5an3eP/+/SY+Pt5lmsmTJ9v5Nm7c2Fy/fj3FdMuWLTOjRo0y8+fPN3v37jWxsbHm6NGj9rFPPPGE198XT8ScqlWrGndf3o5/GX35ZI05M8LDw9NMk9iscOedd1K8eHFatmyZYjPEyJEj7eaLYcOGpdj2+9Zbb/HAAw/QrFkzmjZtyoMPPsjSpUuTpXOnlrhq1So+/fRTXn75ZQA6derEpk2b0jzO055//nmP5OPqsjmzTp06xfjx4wEYMGAARYsW9dq5vMFT73G1atXSTPPcc8+xdOlS9u/fT1RUFFu3bqV58+bJ0rVv35727dt7pFzZyR9qzLl2EiPnAJ5Sm3RwcDAPPPAAAJcuXXLZVvrJJ5/Yy5kNRmmVSzmMGjWKK1euUKtWLZ566qnsLo7Pu/322+3ls2fPZmNJvC9Pnjxuv3yV39WY3eX8QT116lSy/W3atKFAgQIA/P7771y7di3VvH755X8TRXXs6PLBBJkul4Iff/yR1atXExAQwNtvv+0XN3u87e+//zdPuzu9M3IyrTF7yQ8//MCJEyeIiYnh3LlzbNu2jf/85z/ceeedHsn/ueeeo2nTpoAj+K1duzZZmjp16tjLW7ZscZnfuXPniIiIAOCWW27J8Ae/UaNGvPDCC4DjRsyiRYsylI8/u3Dhgt1zoGfPnkn+Tipl33zzjd0MV7JkSRo2bJjGETlbQECA2y9f5ZM15k6dOtnLiV3J6tevz4ABA5gxYwb9+/e3+3u6cvfddxMSEgJAYGAglStX5qGHHuLuu+8GHL07+vTpw40bN5IdW716dXs5Mei6cuTIESpXrmwf6+pysWbNmnb+efPmpUyZMrRu3ZquXbuSL18+wNF2vX379jTPm9u8//77REZGUrZsWQYOHJjdxfEpmzdvtvsc37hxg+PHj7N69Wq7YlGgQAFGjx5N/vz5s7OYXucPNWafCsznzp3jl19+YcuWLZw4cQIRsYNpy5YtAXj66aepWLEiHTt2TLOf7wcffJDiTY64uDhWrFjB0KFDUw1+xYoVS1KutDgPMHE+NiVPPfUUr7/+eor7tm/fzujRo5k/f36a58xt/vjjDxYvdsxJPmLECAoVKpTNJfIt48aNS7G7ZkBAAC1atGDw4MHUqlUrG0qWtXy57dhdPhOYhw4dSlhYGHFxccn2vf/++3Tp0oXZs2dTqFAh2rdvz5AhQxg9enSGznXkyBGWLVvG0aNHU00THBxsL7tTO3dugy5cuHCGyhUVFcWyZctc9oXOrRL74QLcd9993HvvvdlcopyjXLlytGzZkrJly2Z3UbKEP9SY3f5qEZEAESknIhUTX54syIYNG1IMyom+++47+vbta6//61//SvOSrEWLFvYjZAoVKkTDhg0ZM2YMZcuW5aOPPmL79u3Z0t42dOhQu1yBgYFUq1aNl156iUuXLvHaa6+xY8cO7Wlwk3//+9+cOHGC4OBghg8fnt3F8Ulz585l37597Nu3j61bt7Jw4UL69u3L2bNnGTt2LF27dmX37t3ZXUyv8+Qz/7KLW4FZRAYAp4HlwI/W6wcvlitFX3/9Nfv2OR5SW6xYMbt5wx3R0dFs27aNYcOG0bJlS6Kiorj11ltZsWJFijWJK1eu2MuJvTNcKViwoL18+fJlt8t148YNDh06xMSJE6lXrx47d+6kQIECzJw50+6ul9tt376dOXPmADBo0CBKly6dxhEqKCiI2rVr88orr/DVV19RqFAhTp06xdNPP52uOWJyIn/oLuduyV4G7jDG1DbG1LVe9VJLLCL9RCRMRMI8U8z/Wb16tb1co0aNDOWxfft2PvjgA8AxnDVxQIezixcv2sslS5ZMM88SJUqkeGx6XLx4kRdffNFeT7x0z81u3LjB8OHDSUhI4M4776RHjx7ZXaQcp1atWjz77LOAo7ksJ0xElBm5qVfG38AldzM1xkwFpoLnn/mXnptsrvz888+8++67ACnOanfgwAF7ObG3hSuVKlVK8dj0Wrt2LVFRURQpUoQmTZrkuDkgPG379u32qMlbb72VTz/9NMV0zgNzDhw4wOTJkwGoUqVKpvuW+4NWrVoxYcIEgGwZTZqVfLmJwl3uBua/gNUi8iMQk7jRGPORV0rlgidqppC0uSGlAO/cFteoUSOXeZUsWdIO3mfOnMn0yKqrV69SpEgR8uTJQ5EiRXJ1YHZ+WPAPP7jXehYeHm4H6nbt2mlghiQ9WKKiorKxJN7ny00U7nI3MB+1XvmtV7Zp06aNvZyZmmnVqlXt5ZS6w61evZrr169ToEABWrduTYECBVLtnXH//ffbyz///HOGywRQpEgRe4BKQkJCmvM8K+UO5x5IxYsXz8aSeF+uqTEbY97ydkHc0b17d2rWrAk4vvXXrFmT4bz69etnL69bty7Z/qtXr7J06VK6detG0aJF6d27N1OmTEkxr5deeslenjt3bobLBI5+2nnzOv4sW7ZsITY2NlP55XTNmjWzb/i68u233zJs2DAgZ0yUn9XmzZtnLzdo0CAbS+J9/hCY3e2VUUpEPhSRpSLya+LLU4UYMGCAPUQ6NQ8//DCfffaZvT5+/HhiYmKSpOnVqxf33Xefy3zy5cvHhx9+yMMPPwxATExMknydvfPOO/YctWPGjKFu3brJ0owcOdIexLJp06YUZ5Zr0aIFzz77LIGBgS7L1qNHD8aMGWOvT5w40WV6lbt9++23rFmzJklzz81u3LjB2LFj+fVXx79rvnz5ePTRR7OqiNkiN938mwPMBR4Cngd6AR6bouree+/lP//5D/v27WPlypXs2bOH8+fP2yP/OnfunKRr3K+//pokgCWqX78+X3zxBX///TfLly9n586dnDlzhhs3bhASEkK9evXo2rUr5cuXt4959dVXU20SSey98frrr1OsWDHWrVvHZ599lmSi/MRmjMuXLyephTu75ZZbmDZtGuPHj2fZsmVs3bqVY8eOcfXqVYKDg6levTqdOnWifv369jHz589n5syZGXo/M+vYsWMsWLAgyTbnydI3btyYbNRlhw4dko0qi4qKYsaMGUm2nThxwl7eu3dvkkdCgeMRXimN1vQ3nniP9+3bx7BhwyhTpgx33XUXd9xxByEhIeTLl49Lly6xf/9+VqxYwZkzZ+xjXnvtNW677bZUy3Xz38P5Xszx48eT7a9VqxYdOnRw4zfOOrmpjbmEMWa6iLxsjPkN+E1ENnu6MDVq1HDZBS4hIYFp06YxaNAgl5f4t956K08//bTLc50+fZqXX345zaaHoUOHEhgYyMsvv0xwcDChoaEp5tWjR480R+wVKVKERx991GWNJSYmhg8//JBRo0a5zMubjh8/nmqzDUBYWBhhYUl7QlasWDHFwOwqn/379yd7OkZAQECuCMyeeo/BMRHXt99+6/J8JUqUYNiwYUnmoUmJqzKdOHEi2f4uXbr4XGD2h6YMdwNzYhQ8KSKdgBNAiKcK8corr/DDDz/QvHlz7rzzTm655Rb7AaoXL17kwIEDrFmzhs8//5yDBw+mms8bb7zBypUradu2LQ0bNqRq1aqULFmSfPnyceXKFU6ePMn27dv56aefWLhwodu9HQYPHsy8efPo168frVu3ply5cly/fp2//vqL7777jsmTJ7u8Sff999/TsmVL2rVrR9OmTbnjjjsoU6aM3RXu/Pnz7N69m9WrV/PVV19x8uTJdL+HKvcJDQ2lefPmbNq0ifDwcI4ePcqFCxeIi4sjKCiIUqVKUaNGDVq3bk2HDh0ICgrK7iJnCX8IzOKqfcpOJPIQ8AdwK/AJUAR4yxizxI1jPdqPWSWXcNOz2pTKqcQDUbV9+/Zux5wVK1b4ZBR3t1dGYgfSS8A93iuOUkpljj/UmN3tlVFdRFaKyG5rvZ6I6EwySimf4w+9Mty9fTkNGIrV1myM2Ql091ahlFIqo/xhdjl3b/4FGWM23fSLpD5Hp1JKZRN/6C7n7m9wTkRuBwyAiDwKaNcBpZTP8XSN2ZqLfpuI/GCtVxGRjSJySETmikh+a3ugtX7I2l/ZKY+h1vb9InJ/ymf6H3cDc3/gU6CGiBwHQoEX3DxWKaWyjBfmY34Z2Ou0Phb4tzGmKnABeMba/gxwwdr+bysdIlILR9NvbaAjMElEXDZwu1UyY8xfxpj2QCmghjGmlTEmwt3fSimlsoona8wiUgHoBHxmrQtwL5A4bHMm0MVafthax9rfzkr/MPCNMSbGGHMYOAS4nIPCZRuziAxOZTuQPdN+KqWUK+npbSEi/QDnuRSmWvPJJ/oYeA1IfJBnCeCiMSbxHtsxIHGOh/I45q7HGBMnIpes9OWBDU55Oh+TorRu/iUWxgA3f73owBGllM9JT28L54d6pJDPQ8AZY8wWEWnrmdK5x2VgTpzuU0RmAi8bYy5a68WB8d4vnlJKpY8Hu8G1BP5PRB4ECuAY8TwBKCYiea1acwXguJX+OI7R0cdEJC9QFDjvtD2R8zEpcrf1u15iUAYwxlwA/HtSV6VUjuSpNmZjzFBjTAVjTGUcN+9+NcY8AawCEmci6wUstpaXWOtY+381jjkvlgDdrV4bVYBqgMvne7nbjzmPiBS3AjIiEpKOY5VSKstkwcCRIcA3IvIusA2Ybm2fDswSkUNAJNYgPGPMHhGZB4TjGP/R3xgTnzzb/3E3uI4H1ovIfGv9H8B76flNlFIqK3hjgIkxZjWw2lr+ixR6VRhjruOIjSkd/x7piJnuTmL0pYiE4egmAtDNGBPu6hillMoOvjzU2l1uN0dYgViDsVLKp/nDkGxtJ1ZK+ZVcVWNWSqmcQAOzUkr5GA3MSinlY7SNWSmlfIzWmJVSysdoYFZKKR+jgVkppXyMBmallPIxGpiVUsrHaK8MN8TF6cO0Vc7nD7Ww3MIf/lZaY1ZK+RUNzEop5WM0MCullI/RwKyUUj5Gb/4ppZSP0RqzUkr5GA3MSinlYzQwK6WUj9HArJRSPkYDs1JK+RjtlaGUUj5GA7NSSvkYbcpQSikfo4FZKaV8jAZmpZTyMdrGrJRSPkZrzEop5WM0MCullI/RwKyUUj5GA7NSSvkYfwjMOf/2pVJKOQkICHD75YqIFBCRTSKyQ0T2iMhb1vYqIrJRRA6JyFwRyW9tD7TWD1n7KzvlNdTavl9E7k/rd9DArJTyKyLi9isNMcC9xpg7gfpARxFpDowF/m2MqQpcAJ6x0j8DXLC2/9tKh4jUAroDtYGOwCQRcfmtoIFZKeVXPBWYjcMVazWf9TLAvcACa/tMoIu1/LC1jrW/nThO8jDwjTEmxhhzGDgENHV17hzVxnz58mXWrFnDpk2bCA8P5+jRo1y9epWgoCDKli1LgwYN6Nq1K3Xr1nWZjzGGI0eOsHv3bvbs2UN4eDjh4eFcvXoVgC5dujB69OgMlfHo0aMsWbKEP/74gxMnThAVFUXRokUpVaoU9erVo1mzZtx3331pXkZlp/j4eP7880/7/dm9ezf79+/n+vXrAPTv358BAwZk6hwjRoxg/vz59np68ty2bRvz589n8+bNnD17lsDAQMqXL899991H9+7dKV68eKbK5mv27t3LggUL2LJlC8eOHSM6Otr+zNevX5+HH36Yxo0bZ3levio9bcwi0g/o57RpqjFmqtP+AGALUBWYCPwJXDTGxFlJjgHlreXywN8Axpg4EbkElLC2b3A6h/MxKcoxgXn69Ol88skn3LhxI9m+qKgooqKi2L9/P9988w2dO3dm1KhRFCxYMMW8PvjgA2bOnJnivoyKj49n4sSJTJ8+ndjY2CT7zp07x7lz59i7dy9z585lw4YNFClSxKPn96TQ0FCWL1/utfw3btzIggUL0k54E2MM77//Pl9++SXGGHv79evXuXTpEuHh4cyZM4dx48bRvHlzTxY5WyQkJDB69Ghmz56d5PcFRyXl8uXLHDhwgHnz5tGpUyfGjBlDYGCg1/PydekJzFYQnupifzxQX0SKAYuAGpkuoBtyTGCOiIiwg/Ktt95K8+bNqVmzJsWKFSMqKooNGzawfPly4uPj+f7774mMjOTTTz9NcXhmQkJCkvVChQpRpkwZ/vzzzwyVLS4ujiFDhvDTTz8BEBISwn333UetWrUoUqQI0dHRHDlyhPXr17N79+4MnSMr3fz+FC1alGLFinHkyJFM5339+nVGjhyJMYagoCCio6PdPvajjz6yv1CDgoJ45JFHqFevHlevXmXZsmWsW7eOc+fO8eKLLzJnzhxq1qyZ6fJmpzFjxjBr1ix7/Z577qFZs2bccsstnD9/nu3bt/Pzzz8THx/Pjz/+SHx8PBMmTPB6Xr7OG70yjDEXRWQV0AIoJiJ5rVpzBeC4lew4cCtwTETyAkWB807bEzkfk6IcE5hFhDZt2vD000/TpEmTZPsfe+wxwsLCeP7554mOjmbt2rV89913dOvWLVna22+/nV69elG7dm1q165N5cqV2bx5M717985Q2SZOnGgH5a5duzJs2DAKFSqULF1oaChnz54lKCgoQ+fJKnXr1uW2226jdu3a1KlThwoVKvDtt98ybNiwTOf93//+lyNHjlC6dGk6duzo9pVLeHg4n332GQCFCxdm9uzZ3HHHHfb+7t2788knnzBx4kSio6MZOXIk8+bNy7Fdp44dO8bs2bMBRy+DqVOn0qpVqyRpnnrqKZ555hl69uxJdHQ0P//8M3v37k32heTJvHICT82VISKlgFgrKBcE7sNxQ28V8CjwDdALWGwdssRaX2/t/9UYY0RkCfCViHwElAOqAZtcnTvHBOZXXnmFokWLukzTuHFjBg0axHvvvQeQamB+7LHHPFaugwcPMn36dADatWtnnzs1pUqV8ti5veX555/3Sr579uzh888/B+CNN95g//79bh87adIk+xJ80KBBSYJyopdeeok//viDnTt3smvXLn777Tfatm3rkbJntfXr19tXLvfdd1+yQJqodu3adO/enRkzZgAQFhaWLJh6Mq+cwINfxmWBmVY7cx5gnjHmBxEJB74RkXeBbcB0K/10YJaIHAIicfTEwBizR0TmAeFAHNDfaiJJVY7plZFWUE50//3/6yJ44MABbxXHNnPmTOLi4hARXn/9da+fL6eKi4tjxIgRxMfHc++999KhQwe3j71y5Qq///47AMHBwXTt2jXFdCJCz5497fXEq5ic6Pz58/Zy5cqVXaZ13n/t2jWv5pUT5MmTx+2XK8aYncaYBsaYesaYOsaYt63tfxljmhpjqhpj/mGMibG2X7fWq1r7/3LK6z1jzO3GmDuMMWl+MHNMjdldzk0IMTExXj3X9evX7X/+hg0bUr68yxutudqMGTMIDw8nKCiIESNGpOvYzZs32/cXGjdunOpNXSBJbfCPP/7IWGF9QIkSJezliIgIl2md9992221ezSsnyKnNV85yTI3ZXQcPHrSXy5Ur59Vz7dmzx65VJHbR27BhA6GhobRt25Y777yTu+++m759+7JgwYJkvTVyi4iICCZOnAg42tnLli2bruOd/6a1a9d2mTYkJMT+u0dGRiapLeYkrVu3Jl++fAAsX76ctWvXpphuz549zJ07F3DUdtu0aePVvHICDw4wyTZ+V2N27hvbunVrr55rz5499nKZMmV49913+eqrr5KkOX/+PGvXrmXt2rV8+eWXTJo0iQoVKni1XL7EGMPw4cOJiYmhTp06PPHEE+nOw7kW585VSbly5Thx4oR9rHONMacoXbo0r776KmPGjCE+Pp6nn36ae+65h+bNm9s9KbZt22b3pKhatSoTJ060A7C38soJdKJ8H7Nt2zYWLVoEQGBgIL169fLq+c6dO2cvf/PNN0RERBAQEMCDDz5Is2bNCAwM5MCBAyxYsIALFy5w6NAhevfuzcKFC91uM8/p5s6dS1hYGAEBAbz99tsZGlhz+fJle9mdwSPOaaKiotJ9Pl/Ru3dvSpUqxbhx4zhx4gSrVq1i1apVSdKEhIQwaNAgOnfu7LKJx5N5+Tpfrgm7K+d/tVjOnj3L4MGD7bvPAwYMoEyZMl49p/M/fUREBIGBgUyfPp2xY8fSrVs3OnXqxKBBg/j++++pXr06ACdOnODjjz/2arl8xenTpxk3bhwATz75JLVq1cpQPokjMgG3Bj04p3E+Nifq0KEDr7/+OqVLl05xf2RkJJ999hlLly7N0rx8mT80ZfhFYI6Ojuall17i9OnTALRp04Y+ffp4/bw3j6B67rnnaNo0+RD4kJAQPvjgA/uDsGjRIq5cuZIsnb956623uHLlCuXKlWPgwIHZXZwc5+jRo3Tp0oWBAweSL18+xo4dy5o1a9i9ezdr1qxh7NixVKhQgSNHjjBs2DDGjx+fJXn5ulwVmEWkkoi0t5YLikhh7xXLfTExMfTv359du3YBjt4R48ePz5I3/eaBIv/4xz9STVu9enXuvPNOAG7cuMHWrVu9Wrbs9tNPP/Hrr78CMHLkyEwNqklvTxvnNCkN9MkJTp8+zWOPPcahQ4eoVKkSCxcupEuXLpQqVYp8+fJRqlQpunTpwsKFC6lYsSIAU6dOZfXq1V7NKyfINYFZRPrimC3pU2tTBeA7F+n7iUiYiIRNmzYt86VMxY0bNxg4cCAbN24EHD0jpkyZkmUj65znuyhTpkyaN5mcL+X//vtvr5Uru128eJF3330XgI4dO2Z6kEfhwv+rA1y4cCHN9M5pfHlOElcmT55s/x6hoaEUK1YsxXTFihUjNOVCD48AACAASURBVDTUXncedu2NvHICfwjM7t78649jmrqNAMaYgyJyS2qJnScGiY+PN6mly4zY2FgGDRpk91WtWbMmU6dOJTg42BunS5FzZ3zn4JEa5zTON7T8zapVq+xuaiEhIUyePDnFdGFhYUmWE9PVq1ePli1b2vuc3+fjx11OMQBg98i4+dic5LfffrOXW7Ro4TKt8/7EK0dv5ZUT+PLMje5yNzDHGGNuJH7DWBN0eCXguiMuLo5//etf9l3l6tWrM3369Czv6eA8LNidQOucxp1AnlM5t73f3H0wNRs3brSvfJ566qkkgblatWr2snMXxZRERkbagTkkJCRHdpUDOHPmjL2cVmXD+bOU0qRQnswrJ/DlmrC73G1j/k1EhgEFReQ+YD7wvfeKlbr4+HiGDBnCsmXLAMeERNOnT0/18sybbrvtNrtP8qlTp9IczBAeHm4v59SaXHZo2rQp+fPnBxw168R5oVOyZs0ae/nuu+/2etm8xTmAnjp1ymVa5yuElP4PPJlXTuAPTRnuBubXgbPALuA5YCkw3FuFSk1CQgLDhw+3h0FXqVKFGTNmZGut6IEHHrCXnQe33OzAgQPs2LEDcNw0bNiwodfLll26devGvn370nz179/fPqZ///729ptnsStUqJA9Cu3KlSt2X/WbGWOYM2eOve78t8lpnK8SfvzxR5dpnffXqVPHq3nlBJ6aKyM7uVuyLsCX1gQdjxpjppmb+4p5mTGGUaNGsXixY4a9ihUr8vnnn2f7bG19+vSxbzB9+umnbNqUfDa/yMhIXnvtNfsSv0ePHjm6A392eOGFF+wazkcffZTizHQTJ060v/zq1q2bY2eWA+jUqZO9PGnSJNavX59iuvXr1zNlyhR7/eGHH/ZqXjmBP9SY3W1j7gz8W0R+B+YCPzs9WiVLfPzxx/ZTL/LmzcuTTz7Jzp070zyuZcuWyYJgVFSUPf1kIudLuL179yabJLxZs2YpPhWjWLFijBw5kn/961/ExMTwzDPP0KlTp2Qj/yIjIwFHu7RzTdEXHTt2LNkTRpwD4caNG4mPTzprYYcOHTI8gMQdtWrV4tlnn2XatGlcvnyZHj168Oijj1K3bl2io6NZtmyZPQdEUFAQb7/9ttfKkhUeffRRFi5cyK5du4iJieHpp5+mffv2tGzZkmLFinHx4kXWrl3LihUr7EFVd999Nx07dvRqXjmBLwdcd4m7FV8RyQc8ADwOtAKWG2OeTes4T/XK6NWrF5s3b073ccuXL082v8Lx48e577770pXPiy++yEsvvZTq/kWLFvHuu++6nCqxSZMmfPzxxx5/Jp2nL8k2btyY7uHso0ePTnHua1cSJ7YH9575l9qjpZyVKFGC8ePHe/zRUtnxz37hwgVeffXVJO3mqenYsSOjR49Otd+2J/Pysky/0TNnznQ75vTq1csno7jbc2UYY2JF5CccvTEK4mjeSDMw5xZdu3aladOmzJs3j99//52TJ09y7do1QkJCqFevHp07d6Zdu3Z+8W2eXUSEoUOH0rFjR+bNm0dYWBhnzpwhMDCQChUq0L59e3r06OE3D2MtXrw406dPZ926dXz//ffs3LmTU6dOce3aNQoWLEi5cuWoX78+Xbp0oVGjRlmWl6/z5bZjd7lVYxaRxJpyW2A1MA9Y5k5zhrf6Mav/8YcPoq/TL9Qsk+k3evbs2W7HnJ49e/rkH9bdGvNTONqWn0ucrV8ppXyRP3yJuhWYjTE9vF0QpZTyBL8PzCKyxhjTSkQuk3SknwDGGJMzJyJQSvktvx+SbYxpZf303/HDSim/4g81Zndnl7tdRAKt5bYiMlBEcuZ4TaWUX/OHASbu3s5fCMSLSFUcs8bdCrg3O41SSmUhfwjM7vbKSDDGxIlIV+ATY8wnIrLNmwVTSqmM8Ifuo+4G5lgR6QH0wjE8GyBnPkJXKeXXfLkm7C53A3Mf4HngPWPMYRGpAuTMxxsopfxarqkxG2PCgYFO64eBsd4qlFJKZVSuCcwi0hIYBVSyjknsx3yb94qmlFLpl5uaMqYDg4AtQHwaaZVSKtvkpsB8yRjzk1dLopRSHpCbAvMqEfkQ+BawJzEyxmz1SqmUUiqDclNgbmb9bOy0zQD3erY4SimVOf4wV4Zbty+NMfek8NKgrJTyOZ4a+Scit4rIKhEJF5E9IvKytT1ERJaLyEHrZ3Fru4jIf0TkkIjsFJGGTnn1stIfFJE0Hw/k7lwZpUVkuvUEE0Sklog8486xSimVlTw4JDsOeMUYUwtoDvQXkVrA68BKY0w1YKW1Do5H71WzXv2AyVZ5QoA3cbQ8NAXeTAzmqXG3w98XwC9AOWv9ABDq5rFKKZVl8uTJ4/bLFWPMycT7aMaYy8BeoDzwMDDTSjYTx2P2sLZ/aRw2AMVEpCxwP45npEYaYy4AywGXT7p1NzCXNMbMAxKsQsah3eaUUj7IG5MYiUhloAGwEShtjDlp7ToFlLaWywN/Ox12zNqW2vZUuRuYr4pICazJ8kWkOXDJzWOVUirLpCcwi0g/EQlzevVLIb9gHDNshhpjopz3GcdDUz3+XFN3e2UMBpYAt4vIWqAU8KinC6OUUpmVnpqwMWYqjqmMU8srH46gPMcY8621+bSIlDXGnLSaKs5Y24/jmBI5UQVr23EcD7J23r7aVblc1phF5B/W4gWgDXAX8BxQ2xiz09WxSimVHTzYK0NwjHrea4z5yGnXEhwzbWL9XOy0/Smrd0ZzHAPzTuK4P9dBRIpbN/06WNtSlVaNeSgwH1hojGkI7EkjvVJKZSsPDjBpCTwJ7BKR7da2YcD7wDyrZ9oR4DFr31LgQeAQEI1jVk6MMZEi8g6w2Ur3tjEm0uXv4GgiSWWnyAocN/yaAr/fvN8Y839p/Wbx8fEeb39RSfnDbFq+zh9Gk+UQmX6j16xZ43bMadWqlU/+YdOqMT8INMQx9/J47xdHKaUyxx++RNMKzNONMU+KyDRjzG9ZUiKllMqE3BCYG4lIOeAJEZnGTZcZabWTACQkJGSieMod/jA3gK+7ceNGdhchV8ifP3+m88gNgXkKjiGHt+GYizmR4Oi7pxPlK6V8it8HZmPMf4D/iMhkHEG6tbXrd2PMDm8XTiml0ssfArO7t/P3AbOBkjgGl8wSkQFeK5VSSmWQN4ZkZzV3R/49AzQ3xlwFEJGxwHrgE28VTCmlMsIfuo+6G5iFpJMWxeOB/oZKKeVpvlwTdpe7gflzYKOILLLWu+AYqqiUUsrD3ArMxpiPRGQ10Mra1McYs81rpVJKqQzKTTXmxAev6sNXlVI+LVcFZqWUygk0MCullI/JTb0ylFIqR9Aas1JK+RgNzEop5WP8ITDn/MYYpZTyM1pjVkr5FX+oMWtgVkr5Fe2VoZRSPkZrzEop5WM0MCullI/xh8Cc8xtjlFLKz2iNWSnlV/zh5l/O/w2UUsrPaI1ZKeVX/KGNOUcF5suXL7N27Vo2bdrE3r17OXr0KFevXiUoKIgyZcrQoEEDunTpQt26dVPN47vvvmP48OHpPnfjxo354osvkm03xrBt2zY2bNjAjh07OHToEBcuXCBPnjwUL16cmjVr0r59ezp27Ej+/PnTfd6cYsuWLfz4449s2rSJM2fOcP36dUqUKEGZMmVo0qQJrVu3pnHjxmnms2/fPpYsWcK6des4deoUV65cISQkhNKlS9OwYUPuuusu2rRpkwW/kfcYY9i+fTu7d+9m9+7dHD58mMjISC5evIiIUKRIEapVq0br1q156KGHKFKkSKp5vfHGGyxZssTtc+/atcutdNu3b2fhwoWEhYVx7tw58ufPT/ny5WnXrh2PPfYYxYsXd/ucWc0fArMYY7x6gtjYWI+cYMaMGfz3v//lxo0baaZ96KGHePPNNylYsGCyfRkNzF26dOHdd99Nsu3PP/+kb9++nDlzJs3jK1euzNixY6ldu3a6z52WfPnyeTxPd0VGRjJq1Ch++eUXl+lq1KjB4sWLU91/7do1xowZw/z580lISEg1XeHChQkLC8tweTPKnc+du2JiYtz6kgIICQnhzTff5N57701xv6cDszGGDz/8kNmzZ5NabChRogRjx46lWbNmbp/XXfnz5890VI2IiHA75lSuXNkno3iOqTFHRETY/xwVKlSgRYsW3HHHHRQvXpyoqCg2bNjAihUriI+P54cffiAyMpIpU6YkuxHQrFkzJkyYkOb5EhISGDp0KNevXwega9euydJcunTJDsoFCxakRYsW1K9fn9KlSwNw4MABvvvuO86fP09ERATPPvsss2bNomrVqpl6L3zFuXPn6N27NwcPHgTg9ttvp3379lSuXJmgoCAuXrzIwYMH+f33313mc/XqVZ5//nk2bdoEQLly5ejQoQPVqlUjODiYy5cv89dff7FmzRpOnTrl9d8rq9xyyy3Uq1eP6tWrU7ZsWQoVKsT169c5fPgwy5Yt48iRI0RGRjJ48GAmTZrEXXfd5TK/kSNHEhISkqkyffzxx8yaNQtwfKa7detGnTp1iI6OZsWKFaxfv57z588zcOBAZs6cSY0aNTJ1PpWyHFNjfvPNNzl79ix9+vShSZMmKabZsmULL7zwAtHR0QC88847KQZUd6xZs4bnn38egEqVKvHjjz8mS7N161aGDh3KM888Q6dOnShUqFCyNFFRUQwcONCu5aXWJJIZ2VFjNsbw5JNPsnnzZgICAhg2bBj//Oc/U70jfvLkScqWLZvivldffZXvv/8egOeff57+/fun2uzjKh9v8mSNOSEhgcOHD3P77benmiY+Pp4xY8Ywd+5cAKpUqZJizdi5xvzzzz9Tvnz5DJdr7969PP744xhjKFy4MJ9//jl33HFHkjSTJk1i8uTJANSpU4evvvrKo00HnqgxHz161O2YU7FiRZ+sMeeYXhmJtYbUgjJAo0aNCA0NtdddXTqnZdGiRfbyww8/nGKaGjVq8MMPP/DYY4+lGJQBihQpwvjx4+1mlbCwMI4dO5bhcvmKb775hs2bNwPw2muv0bNnT5fdlFILpr///rsdlJ966ikGDRrksi0+O4Kyp+XJk8dlUAYICAhgyJAhFCtWDIDDhw/z999/e7VcU6ZMsZsvBg4cmCwoA7zwwgv2PZzdu3fzxx9/eLVMuVWOCcxFixZ1K12HDh3s5QMHDmToXJcuXWLVqlWA4x8ktcAcFBTkVm21RIkSNGrUyF5PvPTPqYwxfP755wBUrFiRp556KsN5TZ8+HYBChQol+VJVjiuhihUr2uvnz5/32rmuXr3KmjVrAAgODk71My8i9OjRw17/+eefvVamjBIRt1++Kse0MbvLueYaExOToTx++OEH+9L1rrvustuMPVWuxHbrnCosLIwjR44AjhutGe3Qf/z4cTZu3AhA+/btU73qyK0SEhI4ceKEvV6iRAmvnSssLMz+zDdq1CjFG+eJWrZsaS+vXbvWa2XKKF8OuO7yu8B86NAhe7lcuXIZyuO7776zlzPaRn0zT5TLVyQ2YQDUq1ePhIQEFi1axKJFizh48CDR0dGULFmSBg0a0K1bN1q1apViPmFhYfalc7169QBYtmwZ8+bNY+/evURFRVGsWDHq1q3LQw89xAMPPOAX/3TuMMbwySefcO7cOcDRbHbrrbe6PGbUqFFERERw/vx5ChQoQKlSpWjQoAEPPfRQmr1AnK/iatWq5TJtSEgI5cqV48SJE0RGRnL+/Hmvfmmklyc/IyIyA3gIOGOMqWNtCwHmApWBCOAxY8wFcZx4AvAgEA30NsZstY7pBSR2B3vXGDPT1Xn9LjDPnz/fXm7dunW6j9+3bx979+4FoHjx4txzzz2ZLtPWrVv5888/AceHuk6dOpnOMzvt3r3bXg4KCqJnz55s2bIlSZoTJ05w4sQJfvzxR+6//37Gjh2brBbmnE9ISAgDBgxg2bJlSdKcOXOGlStXsnLlSubMmcMnn3yS6Z4HvmbNmjX21d3169c5evQoK1euZP/+/QAUK1aMt956K818NmzYYC/HxsbavVkWLlxI69atGT16dKpNgolXQOBexaFs2bJ2bf7IkSM+FZg97Avgv8CXTtteB1YaY94Xkdet9SHAA0A169UMmAw0swL5m0BjwABbRGSJMeZCaif1q8C8bds2u7YbGBjIk08+me48nG/6derUKdM9HmJjY5P0f+7Tpw8BAQGZyjO7JdbiwNFFKyIigiJFivDoo49Sq1Yt4uLi2Lx5M0uWLCE2NpZffvmF2NhY+25+SvlMmDCBiIgIAgMD6datG/Xr1ydPnjzs2rWLBQsWEB0dTVhYGH379uXrr7/2q8E6w4cPT7H9OF++fLRt25bBgwdToUKFVI8vVKgQLVq0oE6dOpQpU4aAgABOnz7NunXrWLduHeC4ydq7d29mzZpFcHBwsjwuX75sL7szeCTxpuTNx/oCT86VYYz5XUQq37T5YaCttTwTWI0jMD8MfGkcl4EbRKSYiJS10i43xkQCiMhyoCPwdWrn9ZvAfO7cOV599VV7cMJLL71EmTJl0pVHbGxskm5xnmjGeOedd+ybkDVr1qRnz56ZzjO7RUVF2csRERFUqlSJL7/8Msn73bVrV7p3706fPn24cuUKv/76K0uXLuXBBx9MNZ/ixYvz5ZdfUr16dXv7//3f/9GzZ0+efPJJTp8+ze7du/niiy/o16+fl3/L7FelShWaN2/u8grhn//8J2+88QZBQUHJ9vXq1YstW7YwePBgIiMjOXToEOPGjWPUqFHJ0iZ2MQXc+tILDAy0l69evZpm+qyUnqYMEekHOH+YphpjpqZxWGljzElr+RSQeBOqPODcdeaYtS217anKMb0yXImOjmbAgAGcPn0acDRh9O7dO935rFq1iosXLwKOdraUugulx4wZM/j2228Bx4i1cePGZesoPU+5ue/7mDFjUvwSrFevHoMGDbLXv/zyyyT7b85n2LBhSYJyokqVKiW5lE8cAOEvVq9eza5du9i5cyfr169n1qxZPP744/z555+88847PPHEE6l2latdu3aKQTlRo0aN+Pjjj+1g9d1339n/JwqMMVONMY2dXmkF5ZuPNziaJzzKrcAsItVFZKWI7LbW64lI+sc1e0FMTAwDBgywh5o2aNCAcePGZegGgHMzRrdu3TJVrvnz5/PRRx8BjhFUkyZNolKlSpnK01c4956oWrVqkq6AN+vWrZv9ZbRz584ktSvnfAoXLpykNn2ztm3bcssttwCOdufENnt/IiIEBwdTv359hg8fzsSJEwkICODQoUP07ds3Sa02PRo0aGCPGoyPj7ebN5w5B3d3BtM493jytd40WdBd7rTVRIH1M3FOhuOA8x3aCta21Lanyt0a8zRgKBALYIzZCXRPLbGI9BORMBEJ++yzz9w8RfrFxsYSGhpqd7mqW7cukydPdlmDSM2ZM2fsD2xgYKDLIJGWxYsX8/bbbwNQoEABJk6cSIMGDTKcn68pXLiwvZzW3B9BQUFUqVIFcASF48f/93l0zueOO+4gb97UW9ZEJMm5jh49mu5y5zQtW7a0+xMfP37cHoiTEc69Mg4fPpxsv/Pf4sKFVO9J2RKvLG8+1hdkQWBeAvSylnsBi522PyUOzYFLVpPHL0AHESkuIsWBDta2VLkbmIOMMZtu2haXWmLny4Nnn33WzVOkT2xsLIMHD7ZHHtWsWZMpU6akeGPDHYsXLyY+Ph6Adu3auZzRy5WlS5cyYsQIjDHkz5+fCRMm0LRp0wzl5asSAy2490/p/DdxvlHkqXz8mXOfYeduiunlfLPOuW0/kfPVnHPf6dScPHnSXva1K0FPBmYR+RpYD9whIsdE5BngfeA+ETkItLfWAZYCfwGHcFRmXwSwbvq9A2y2Xm8n3ghMjbs3/86JyO1YbSki8ihw0vUh3hMXF8drr71mj86rVq0a06ZNc3t0YEqch29n9KbfL7/8wtChQ0lISCBfvnx89NFHSf6x/IVz27s7AfLKlSv28s21ZE/k48+cmwky82XkXMNNqdJRrVo1ezk8PNxlXpGRkXbwDgkJ8bmucp7sx2yM6ZHKrnYppDVA/1TymQHMcPe87gbm/sBUoIaIHAcOA0+4exJPio+P5/XXX2f58uWAY0azzz77LEmNIL22bt1KREQE4OjD2bx583TnsXLlSoYMGUJ8fDx58+Zl3LhxtG3bNsNl8mXO/cP37NnjMm10dLR96ZwvX74k3b6aNGlCUFAQ0dHR7N+/n7i4uFSbM4wxSQJG5cqVM/Eb5BzOTTaZ+Yw7T5WaUg23SZMm5M+fnxs3brBlyxauX79OgQIFUszLebSfL1Y8/GEQkrtNGcYY0x4oBdQwxrRKx7Eek5CQwIgRI+zx+VWqVGH69OmZ/sa+ecKi9P5hf//9d1599VXi4uIICAhg7NixtGuX7AvVb5QvX95uMz906FCywSXOvv32W2JjYwFo2LBhkvb/ggUL2vMMX758maVLl6aaz+rVq+3eBBUqVEjSDOKvEkdUJqpfv36G8tm2bZt9/yRPnjwpBtOgoCDuvvtuwHFlktoEYMYYvvnmG3u9Y8eOGSqTcs3d4LoQwBhz1RiTeD21wDtFSpkxhrfeesue4rBixYpMnz6dkiVLZirf6Ohoe5J3EUl3M8b69esZNGgQsbGxBAQEMGbMGO6///5MlSknePnll+3loUOHptgFa+fOnfz73/+215955plkafr372/XkkePHp3iBE9Hjx5N0vc2pXxyklmzZrFjxw6Xaa5evcrQoUPtUahFixblgQceSJIm8Ukvrqbu3bp1K6GhoXaazp07p9q//7nnnrMrJRMmTLBHHjqbMmUKO3fuBBzTfmZkdK23+f0kRiJSA6gNFBUR5/5jRYCUr3O8ZMKECSxcuBCAvHnz0rNnT7cek3PXXXe5nJBl2bJldjekZs2apWsei3379jFw4EC761D79u0JDAxk5cqVLo+rUqUKt912m9vn8UUtWrSgR48efP311xw5coSHHnqIf/zjH0lG/i1evNiuLT/22GMpPhLqtttuIzQ0lHHjxnHhwgUeeeQRHnnkEXvk386dO+2RfwCtWrWie/dUOwTlCJs3b+aDDz6gUqVKNG3alKpVq1K8eHHy5MnDhQsX2Lt3LytXruTSpUuA4/P+1ltvJWvK2Lt3L7Nnz6ZMmTK0bNmSatWq2fmcPn2a9evXJwncVatW5bXXXku1XDVr1qRPnz7MmDGDy5cv8+STT/LII48kmSg/seYdFBTEm2++6aV3KHN8OeC6K6025jtwTOBRDOjstP0y0NdbhUrJ9u3b7eW4uDhGjx7t1nG//PKLy8nDnS8V01tb3rdvH9euXUtyrrQesQSOOW3790/xHkGOMnLkSAICApgzZw5RUVH2FJ43e/LJJxk6dGiq+fTt25f4+Hj++9//EhMTw1dffcVXX32VLF3Hjh15//33/eLx9OCYY8J5joqUVKhQgZEjR9KiRYtU05w6dcqutKSmXbt2jBo1Ks3eRqGhody4cYM5c+Zw7do1Zs+enSxNSEgIH3zwgc8+vcTvA7MxZjGwWERaGGPWZ1GZsszRo0ft9tEiRYrQvn37bC5RzpInTx5GjBhB586dWbBggf0gVoDSpUvTpEkTevTo4dZzDp9//nnat2/P3LlzWbt2LadOnSIuLo6SJUvSsGFDHnnkEZfBKSd55513WL9+PVu2bGH//v0cO3aMixcvYoyxHyxco0YN7rnnHtq0aZPqaNHevXtTq1YtduzYwd69ezl//jwXL14kJiaGwoULU758eerXr0/nzp3TnDEukYgwZMgQ7r//fhYsWMCWLVs4e/Ys+fPnp0KFCtx77708/vjj+jBWL3Pr0VIiUgB4Bkezht2EYYx5Oq1jPfVoKZU6fxjm7es8+WgplTpPPFrq0qVLbsecokWL+mQUd/eacBZQBrgf+A3HkMLc0cNfKZWj+MPNP3cDc1VjzAjgqjXBcycc840qpZTyMHcHmMRaPy+KSB0cU93d4p0iKaVUxvlyTdhd7gbmqdbkG8NxTNQRDIzwWqmUUiqDclNgngU8guMZV4nPqsr8E0qVUkol425gXgxcArYAGXv0tFJKZYHcVGOuYIzRQfFKKZUF3O2VsU5E6nq1JEop5QH+0F0urbkyduGYgzkv0EdE/sLRlCE4Zpyr5/0iKqWU+3w54LorraaMh7KkFEoppWxpzZXheoYVpZTyMf5QY/aPabqUUsqPaGBWSikf4253OaWUyhH8oSlDA7NSyq/4Q2DWpgyllPIxWmNWSvkVrTErpZTyOK0xK6X8itaYlVJKeZzWmJVSfsUfaswamJVSfsUfArM2ZSillI/RGrNSyq9ojVkppZTHaY1ZKeVX/KHGLMaY7C6DzxGRfsaYqdldDn+m77H36Xucc2lTRsr6ZXcBcgF9j71P3+McSgOzUkr5GA3MSinlYzQwp0zb5bxP32Pv0/c4h9Kbf0op5WO0xqyUUj5GA7NSSvmYXBmYRWSUiLya3eXIyURkoIjsFZE5IhIoIitEZLuIPO7Bc6zzVF7+REQqi8judKTvIiK1vFkm5Vk68k9l1ItAe2PMMRFpDmCMqe/JExhj7vJkfrlYF+AHIDy7C6LckytqzCLylIjsFJEdIjLrpn23i8jPIrJFRP4QkRrW9s4islFEtlm1wdLW9lEiMkNEVovIXyIyMDt+p6wkIoNFZLf1ChWRKcBtwE8iMgSYDTSxasy3i0gjEfnNek9/EZGyVj6rRWSsiGwSkQMicre1vba1bbv1d6pmbb9i/fxGRDo5lecLEXlURAJE5EMR2Wwd91xWvzfZKEBEponIHhFZJiIFRaSv9V7sEJGFIhIkIncB/wd86PT3SfEzr3yIMcavX0Bt4ABQ0loPAUYBr1rrK4Fq1nIz4FdruTj/67XyLDDeWh4FrAMCgZLAeSBfdv+eXnz/GgG7gEJAMLAHaABEOL2nbYEfrOV81vtTylp/1fnMZQAAApFJREFUHJhhLa92eh8fBFZYy58AT1jL+YGC1vIV62dXYKbT/r+BgjhGtg23tgcCYUCV7H7PsuBvUhmIA+pb6/OAnkAJpzTvAgOs5S+AR532pfiZ15fvvHJDU8a9wHxjzDkAY0xk4iQnIhIM3AXMd5r4JND6WQGYa9X28gOHnfL80RgTA8SIyBmgNHDM279INmkFLDLGXAUQkW+Bu12kvwOoAyy33tMA4KTT/m+tn1twBBiA9cAbIlIB+NYYc/CmPH8CJohIINAR+N0Yc01EOgD1RORRK11RoBpJ/1b+6rAxZru1nPhe1hGRd4FiOL5Ef7n5oDQ+88pH5IbA7Eoe4KJJuW30E+AjY8wSEWmLo6acKMZpOR59H50JsMcY0yKV/Ynvnf2+GWO+EpGNQCdgqYg8Z4z5NfEAY8x1EVkN3I+jBv6N07kGGGOSBaBc4ObPYEEcNeMuxpgdItIbx5XMzVx95pWPyA1tzL8C/xCREgAiEpK4wxgTBRwWkX9Y+0RE7rR2FwWOW8u9srC8vuYPoIvVXlkIR7PCHy7S7wdKiUgLABHJJyK1XZ1ARG4D/jLG/AdYDNRLIdlcoA+O2vrP1rZfgBdEJJ+VT3WrjLlVYeCk9X484bT9srUvrc+88hF+H5iNMXuA94DfRGQH8NFNSZ4AnrH27QEetraPwnG5twU4l0XF9TnGmK04amKbgI3AZ8aYbS7S3wAeBcZa7+l2HJfOrjwG7BaR7TiaQb5MIc0yoA2Odukb1rbPcPQ02Gp1H/uU3H31MgLH32gtsM9p+zfAv6wb2beT+mde+Qgdkq2UUj7G72vMSimV02hgVkopH6OBWSmlfIwGZqWU8jEamJVSysdoYFZKKR+jgVkppXzM/wM/G4D7YwlTAwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JtFNBYbxdKFO"
      },
      "source": [
        "## BERT4News"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 754
        },
        "id": "fZ9AOzuDAcCg",
        "outputId": "f66f1a1f-2735-4f3a-b632-9653b46fee6f"
      },
      "source": [
        "# Bert4news\n",
        "from sklearn.metrics import f1_score, confusion_matrix, accuracy_score, recall_score\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments, BertTokenizer, BertForSequenceClassification\n",
        "\n",
        "import seaborn as sn\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "train_X, train_y = pre_process_features(X_train, y_train, tokenized=False, lowercased = False)\n",
        "dev_X, dev_y = pre_process_features(X_dev, y_dev, tokenized=False, lowercased = False)\n",
        "test_X, test_y = pre_process_features(X_test, y_test, tokenized=False, lowercased = False)\n",
        "  \n",
        "model = AutoModelForSequenceClassification.from_pretrained(\"NlpHUST/vibert4news-base-cased\", num_labels = 3)\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"NlpHUST/vibert4news-base-cased\",use_fast=False)\n",
        "\n",
        "import torch\n",
        "\n",
        "class BuildDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "train_encodings = tokenizer(train_X, truncation=True, padding=True, max_length=100)\n",
        "dev_encodings = tokenizer(dev_X, truncation=True, padding=True, max_length=100)\n",
        "test_encodings = tokenizer(test_X, truncation=True, padding=True, max_length=100)\n",
        "\n",
        "train_dataset = BuildDataset(train_encodings, train_y)\n",
        "dev_dataset = BuildDataset(dev_encodings, dev_y)\n",
        "test_dataset = BuildDataset(test_encodings, test_y)\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='drive/MyDrive/CODE/HSD/transformer_model/bert4news-v8/',          \n",
        "    num_train_epochs=2,              \n",
        "    per_device_train_batch_size=16,  \n",
        "    per_device_eval_batch_size=16,   \n",
        "    warmup_steps=500,                \n",
        "    weight_decay=0.01,\n",
        "    no_cuda=False,\n",
        "    do_eval=True\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,                         \n",
        "    args=training_args,                  \n",
        "    train_dataset=train_dataset,         \n",
        "    eval_dataset=dev_dataset             \n",
        ")\n",
        "\n",
        "trainer.train()\n",
        "trainer.save_model(\"drive/MyDrive/CODE/HSD/transformer_model/bert4news-v9/\")\n",
        "\n",
        "y_pred_classify = trainer.predict(test_dataset)\n",
        "\n",
        "y_pred = np.argmax(y_pred_classify.predictions, axis=-1)\n",
        "y_true = test_y\n",
        "\n",
        "cf2 = confusion_matrix(y_true, y_pred)\n",
        "print(cf2)\n",
        "\n",
        "evaluation = f1_score(y_true, y_pred, average='micro')\n",
        "\n",
        "print(\"F1 - micro: \" + str(evaluation))\n",
        "\n",
        "evaluation = f1_score(y_true, y_pred, average='macro')\n",
        "print(\"F1 - macro: \" + str(evaluation))\n",
        "\n",
        "# Show out the confusion matrix\n",
        "df_cm2 = pd.DataFrame(cf2, index = [\"clean\",\"offensive\",\"hate\"],\n",
        "                  columns = [\"clean\",\"offensive\",\"hate\"])\n",
        "plt.clf()\n",
        "sn.heatmap(df_cm2, annot=True, cmap=\"Greys\",fmt='g', cbar=True, annot_kws={\"size\": 30})"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at NlpHUST/vibert4news-base-cased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at NlpHUST/vibert4news-base-cased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='3006' max='3006' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3006/3006 12:09, Epoch 2/2]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.557800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.512500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.482600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.425000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.399400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.373200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='418' max='418' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [418/418 00:21]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "[[5347   57  144]\n",
            " [ 237  115   92]\n",
            " [ 302   49  337]]\n",
            "F1 - micro: 0.8681137724550898\n",
            "F1 - macro: 0.6052139449018336\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7ffa186d0c50>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWYAAAD4CAYAAADfPUyRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xUVfr48c9DEkqC9N4RUHoTMUgRVMqy0qzrssoPVGS/iijqCgqiIIJYKcICgqAootJRlI6VKr33bhCS0AKp5/fH3Lk7gUxyk0ySyeR5v1555dYz505unjlz2hVjDEoppfxHvpzOgFJKqeQ0MCullJ/RwKyUUn5GA7NSSvkZDcxKKeVngrP6BUREu31kMe1Zk/WSkpJyOgt5Qr58+SSzaaQn5hhjMv16WUFLzEop5WeyvMSslFLZScQvC8HpooFZKRVQ8uXL/RUBGpiVUgFFA7NSSvkZrcpQSik/o4FZKaX8jAZmpZTyMxqYlVLKzwQFBeV0FjJNA7NSKqAEQok59/crUUopDyLi+MdBWkdFZIeIbBWRTda2EiKyXEQOWL+LW9tFRMaJyEER2S4iTT3S6WUdf0BEeqX1uhqYlVIBxZeB2dLOGNPYGNPMWh8ErDTG1AJWWusAfwNqWT99gUlWfkoAw4A7gObAMHcw90YDs1IqoGRBYL5eN2CmtTwT6O6x/TPjsg4oJiLlgY7AcmNMpDEmClgOdErtBbSOWSkVUHzc+GeAZdaMdZONMVOAssaYM9b+P4Gy1nJF4ITHuSetbd62e6WBWSkVUNJTEhaRvriqHdymWMHXrZUx5pSIlAGWi8hez/ONMSYrpjbWwKyUCijpCcxWEJ6Syv5T1u+zIjIfVx1xhIiUN8acsaoqzlqHnwIqe5xeydp2Cmh73fY1qeVL65iVUgHFV3XMIhImIje5l4EOwE5gEeDuWdELWGgtLwIet3pnhAMXrCqPH4EOIlLcavTrYG3zSkvMSqmA4sN+zGWB+VZ6wcCXxpgfRGQj8LWIPAEcAx62jv8e6AwcBGKA3gDGmEgRGQFstI4bboyJTPUasvqxRPpoqaynj5bKevpoqezhi0dLlS9f3vE/xJkzZ/xyNIqWmJVSAUWHZCullJ8JhCHZGpiVUgFFA7NSSvkZDcw+tHr1atq2bevo2KNHj1K9evUU9zVp0oTw8HCaNWtGgwYNKF26NKVKlSI4OJioqCh2797N8uXLmTFjBhERERnO7+DBg3n77bft9RkzZtC7d+8Uj61atSpHjx7N0Ov460322GOPsWHDBkfHVqxYkVWrVt2wfd68eQwePDjdr928eXM+//zzdJ/nTxITEzl06BC7du1i165d7Ny5k3379nHt2jUAnnnmGZ599tlMvcbrr7/ON998Y69nJs3Jkyfz0Ucf2evdu3dn1KhRmcpfVvHX/5n08JvA7Cvff/895cqVS3Ff+fLlKV++PPfccw+vvfYaL774IlOnTk33a9xyyy28/vrrmc1qmg4fPpzlr5EbVapUKaezkGkvvPACy5cvz7L0N2zYwLfffuuTtI4cOcLEiRN9klZ20Ma/LNK9e/dU98fExKS6/6+//mLdunVs27aNI0eOcOHCBfLnz0/NmjXp3r07TZs25aabbmLKlCkkJCTw6aefpit/U6dOpWDBgly+fJnChQunefzZs2fTvCa3oUOHcttttwGkO1855eOPP051f8GCBVPcHh4enua54Oqq9vLLL9ulyQceeCD9mfQz13e/K1q0KMWKFePYsWOZTvvatWsMHToUYwyhoaFp/r+kxhjD0KFDiYuLy3Ra2UVLzFlk4cKFaR/kxT333MPu3bu97h8xYgSDBg2yv4a9//77fPHFF8TFxTlKv1+/frRp04bLly8zZswYhg8fnuY5V69edXRNRYsWZfbs2YDrq+6MGTMc5Smn3XvvvRk6r0KFClSoUCHN43766Sc7KFerVo1mzZqlcYb/a9CgATfffDP16tWjXr16VKpUifnz5/Pqq69mOu0JEyZw/PhxypYtS6dOnZg5c2baJ3nx1VdfsXnzZkJDQ+nTpw8TJkzIdP6ymgZmP5RaUHYbPXo0//jHP2jUqBHFixenZcuWrF69Os3zKlasyOjRowFX/V1kZKqDd9Ltn//8J4UKFQJg5cqVnDx50qfp51Zz5861l3v06JGDOfGdp59+OkvS3bVrl/2B/uqrr7J///4Mp/Xnn3/ywQcfANC/f3+KFi3qiyxmuUAIzHl2rgzPAO6tTvp6kyZNomjRovzxxx+MHTvW53nybDycPn26z9PPjaKjo+2Gw6CgoIAJzFkhISGBoUOHkpiYyN13302HDh0yld6bb77J5cuXqVOnDo899piPcpn18uXL5/jHXwVcidmpGjVq2Mt//vlnmsc/8sgjdOnShYSEBJ566imfD9GtV68et99+OwCRkZHMnz/fp+nnVosXL7armVq2bEnZsmXTOCPv+vTTT9mzZw+hoaEMGTIkU2l99913rFmzhqCgIEaMGJGrGtQCocTsl4F5yZIlNG3alJIlS3Lp0iVOnDjBzz//zLRp09i2bVum03/66adp3rw54ArKv/76a6rHlyhRgnHjxgEwfvx4/vjjj0zn4Xp9+vSxl7/88kvHdd7+oG/fvuzevZvo6GjCwsIoV64czZo148EHH6ROnTqZSnvevHn2ciA0+mWVo0eP2g2pAwYMoHz58hlOKyoqyu4K2rNnT+rVq+eTPGaX3PQh4o1fBua///3v9nLJkiUpWbIkjRs3pn///kyfPp1nnnnGbgxKTevWrSlRogQABQoUoFq1atx33320bt0acPXu6N27d5pB8MMPP6RMmTIcP36coUOHZuLKUhYcHMy//vUvez23VWOsXbvWXo6OjiY6Opq9e/cya9Ys7r//foYNG+a1Z0Zq9u7da1c5FS9enLvvvttneQ4k7p4TsbGx1K9fn549e2YqvdGjRxMZGUn58uV57rnnfJTL7KMlZh87d+4cP/74I5s3b+b06dOIiB1MW7ZsCbhKllWqVKFTp04kJiammt6YMWMIDw+/YXtCQgIrVqxg8ODBbN26NdU0OnTowOOPPw7As88+y5UrVzJ4dd7dd999lClTBoCtW7eyZcsWn79GVihWrBitWrWifv36lClTBmMMp06dYvXq1fY1zJs3jzNnzvDJJ58QHJy+282zH26XLl3Inz+/T/MfKL7++ms2bdpEUFAQb775ZqZKjL/88guLFi0CYMiQIYSFhfkqm9nGn+uOnfKbwDx48GA2bdpEQkLCDftGjx5N9+7dmTVrFmFhYdx777288soryUbepcexY8dYtmwZx48fT/W40NBQJk+eDLh6BixevDhDr5cWz2qM3FJaHjhwIPXr1yckJOSGfU8//TTLly/n5Zdf5urVq/z+++9MnTqVf//7347Tj4uLS/Z+azVGyiIiInjvvfcA12jMunXrZjitmJgYhg0bBkD79u1z7TeUQCgxO/5oEZEgEakgIlXcP77MyLp161IMym4LFizgqaeestdffvnlNEtQLVq0sJ9UEBYWRtOmTRk1ahTly5fngw8+YOvWrTRt2tTr+W+//TbVqlXjwoUL9O/fP/0X5YC7rylAbGwsX3zxRZa8jq81adIkxaDs1r59e0aMGGGvT5s2LV315qtWrSI6OhpwNYzWrl0745kNYMOHD+fy5ctUqFAh0/foRx99xOnTpylcuHCmGw9zUjY8JTvLOQrMItIfiMD12O3vrJ8lWZivFM2ePZu9e13PQixWrJhdveFETEwMW7Zs4dVXX6Vly5ZcvHiRypUrs2LFihQbSu644w57XoFXX32VM2fO3HCMLzz++ON2gFu4cKHP+0bnpC5duthzmly6dInNmzc7Ptez77KWllO2dOlSuyvh0KFDCQ0NzXBaW7dutQsFzz//vF21lhsFQnc5pzkbANxqjKlnjGlg/TT0drCI9BWRTSKyyTfZ/J81a9bYyxktRW3dupUxY8YArkalAQMGJNsfEhLCtGnTCAoKYt26dUyaNCnD+U1LoPdddvd+Aedzf0RERNg9ZQoUKECXLl2yJG+5WXR0NCNHjgSgU6dOjicAS0lcXBxDhgwhKSmJhg0b8uijj/oolzkjKCjI8Y+/clrHfAK44DRRzyfP+vrRUufPn7eXixUrluF0fvjhB9566y2AG27qFi1a2F2EDh065HWYbJMmTezlhg0b8tprrwGwb98+RxPIhIeH293JTpw4wbJly9J9Hf6uePHi9vKlS5ccnbNgwQK7Ybd9+/YUKVIkS/KWm61Zs8b+XyhevLjXwsOmTZuSLbuPa9iwof2Nc+vWrRw6dAiAKlWq2O0q19uzZ4+9vG/fPjut6tWr29Vx/sCfqyicchqYDwNrROQ7INa90RjzQZbkKhUlS5a0l911kBnhGSSuD/Cef1inXY+aNm1q11cvWLDAUWD2LC3PnDkzIJ/dFxUVZS/fdNNNjs7Rvstp87xX3POrpGX9+vWsX78ecDUUplQVuGSJsxrKPXv22IH6nnvu8avA7M9VFE45DczHrZ/81k+Oueuuu+zlzMwDULNmTXv53LlzmcpTRhQqVIhHHnkEcM00lltmkkuvjRs32sve5tD2tGnTJnvu6ooVK9KiRYusypoKUHmmxGyMeTOrM+LEP/7xD/ur/8WLF/nll18ynFbfvn3t5d9++y3ZvrVr1zr64/bq1cueMCa1ifJT8sADD9iTwvz0008BOffykiVL7OsKCwuzpzNNjWdpuUePHgHxT5YVevTo4WjekAkTJtgjAr1NlN+8efNk1RTeeM5+pxPlZy2nvTJKi8i7IvK9iKxy//gqE/3790/WSJSSbt268cknn9jr77//PrGxscmO6dWrF+3bt081nZCQEN599126desGuLqoeaabXXJj32W3zz77LM2h8StWrEjW5apPnz4UKFAg1XNiYmJYunQp4Prnuv/++zOfWZXn5KXGvy+AOcB9QD+gF/CXrzJx9913M27cOPbu3cvKlSvZtWsX58+ft0f+denSJVl92KpVq1L8tG7cuDEzZszgxIkTLF++nO3bt3P27Fni4uIoUaIEDRs2pEePHlSsWNE+56WXXspUlUhGVKtWza6SuXDhgs+eNJFd1q1bx8iRI6levTotWrSgZs2aFC9e3B75t2rVqmSjF++4445k31C8+eGHH+yJ2MPDw5P9nQLNyZMnb/i7e96HKfXr79ChQ6YGkOQVeamOuaQxZpqIDDDGrAXWisjGNM9Kp9q1a6faBS4pKYmpU6fywgsvEB8f7/W4ypUrJyuRpiQiIoIBAwYwZ86cDOc3o3r37m3fPHPmzOHq1avZngdfOHLkCEeOHPG6X0R4+OGHGTx4sKPh1Hmp7/Lp06e99n4A2Lx58w39vqtWraqB2YFAqMpwGpjdUfCMiPwdOA2U8FUmXnzxRZYsWUJ4eDiNGjWiTJky9gNUo6Oj2b9/P7/88guffvopBw4c8JrOa6+9xsqVK2nbti1NmzalZs2alCpVipCQEC5fvsyZM2fYunUrS5cuZe7cuTnymBwRoVevXvZ6bqvGABg0aBDt2rVj69at7N27l8jISKKiokhISKBIkSJUq1aN2267jfvvv99Rgx+4hsm7u3YVKVIk03MJq7wrEAKzOOmiJSL3AT8DlYHxQBHgTWPMIgfnBl4fMD8TiN3s/I2v599WKcuXL1+mo+q9997r+B9ixYoVfhnFnfbKcHduvAC0y7rsKKVU5gRCidlpr4xbRGSliOy01huKSO6d5UQpFbACoVeG0+bLqcBgrLpmY8x24B9ZlSmllMqoQJhdzmnjX6gxZsN1F+J9jk6llMoheam73DkRqQEYABF5EMiaeTCVUioT/Lkk7JTTj5ZngMlAbRE5BTwPOH8chVJKZRNfz8dsPSRki4gssdari8h6ETkoInNEJL+1vYC1ftDaX80jjcHW9n0i0jHNa3CSMWPMYWPMvUBpoLYxppUx5qijq1JKqWyUBXXMAwDPyUTeAT40xtQEooAnrO1PAFHW9g+t4xCRurja5OoBnYCJIpJqy2OqVRkiMtDLdiBnpv1USqnU+LK3hYhUAv4OjAQGiiv43Q380zpkJvAGMAnoZi0DfAtMsI7vBnxljIkFjojIQaA58Lu3102rjtk9ga4Brv940VENSim/4+M65o+A//C/WFgSiDbGuDs/nATck7pUxPVQEYwxCSJywTq+IrDOI03Pc1KUamB2T/cpIjOBAcaYaGu9OPC+o8tSSqlslJ7ALCJ9Ac8ZtqZYT2Byj3g+a4zZLCJtfZrJNDjtldHQHZQBjDFRItIktROUUionpCcwez4GLwUtga4i0hkoiGsqirFAMREJtkrNlYBT1vGncE1bcVJEgoGiwHmP7W6e56TIaa+MfFYpGQARKYHzoK6UUtnGV41/xpjBxphKxphquBrvVhljegKrgQetw3oBC63lRdY61v5VxjWRzSLgH1avjepALWBDaq/tNLi+D/wuIt9Y6w/hqgxXSim/kg0DTF4BvhKRt4AtwDRr+zTgc6txLxJrdLQxZpeIfA3sxjUw7xljTGJqL+Bodjmwu3zcba2uMsbsdnieNhJmMZ1dLuvp7HLZwxezyz366KOO/yFmz57tl6NRHFdHWIHYUTBWSqmckpeGZCulVK4QCEOyNTArpQKKBmallPIzGpiVUsrPaB2zUkr5GS0xK6WUn9HArJRSfkYDs1JK+RkNzEop5Wc0MCullJ/RXhkOxMfHZ/VL5Hk6V0bWC4RSWF4RCH8rLTErpQKKBmallPIzGpiVUsrPaGBWSik/o41/SinlZ7TErJRSfkYDs1JK+RkNzEop5Wc0MCullJ/RwKyUUn5Ge2UopZSf0cCslFJ+RqsylFLKz2hgVkopP6OBWSml/IzWMSullJ/RErNSSvkZDcxKKeVnNDArpZSf0cCslFJ+RgOzUkr5maCgoJzOQqbl/n4lSinlQUQc/6SRTkER2SAi20Rkl4i8aW2vLiLrReSgiMwRkfzW9gLW+kFrfzWPtAZb2/eJSMe0rkEDs1IqoPgqMAOxwN3GmEZAY6CTiIQD7wAfGmNqAlHAE9bxTwBR1vYPreMQkbrAP4B6QCdgooikWqzPVVUZly5d4tdff2XDhg3s3r2b48ePc+XKFUJDQylXrhxNmjShR48eNGjQINV0du/ezbZt29i1axf79+8nKiqKqKgoEhMTKVKkCDVq1KBFixZ0796d0qVLe01nw4YN9O7dO93XUaFCBZYvX57u87JLYmIihw4dYufOnezatYudO3eyb98+rl27BsAzzzxD//7900zHGMPRo0ftdHbt2sXu3bu5cuUKAN27d2f06NGO8jR+/Hg+/vhjx9ewYsUKKlWq5Ph4f7dv3z6++uorNm7cyJkzZ0hISKB06dI0btyY7t2706pVq1TPv3btGr/99hvr169nx44dHD16lEuXLpE/f37Kli1Lo0aN6Nq1Ky1atMimK8o6vqpjNsYY4LK1GmL9GOBu4J/W9pnAG8AkoJu1DPAtMEFcmekGfGWMiQWOiMhBoDnwu7fXzjWBedq0aUyYMIG4uLgb9l28eJGLFy+yf/9+5syZQ5cuXRg2bBiFChVKMa1+/fpx/vz5FPedO3eOc+fOsX79eqZMmcLLL7/Mww8/7NNr8feA8fzzz/vkg+Odd95hxowZmc9QHpaQkMC7777LzJkzb9h38uRJTp48yZIlS+jcuTOjRo2iQIECNxy3ePFihg0bRkxMzA374uPjOXz4MIcPH2b+/Pm0bt2ad955hxIlSmTJ9WSH9ARmEekL9PXYNMUYM8VjfxCwGagJfAwcAqKNMQnWISeBitZyReAEgDEmQUQuACWt7es8XsPznBTlmsB87NgxOyhXrlyZ8PBwateuTfHixblw4QLr169n+fLlJCYmsnjxYs6fP8/kyZO9Ds8sXrw4DRs25NZbb6VSpUoULlyY+Ph4jh8/zqpVq9izZw8xMTG8+eabBAcHc//999+QRq1atRg3bpyj/L/99tv8+eefAPTo0SOD70L2SEpKSrZetGhRihUrxrFjxzKVTlhYGOXLl+fgwYOZyt+AAQOoVatWqseULFkyU6/hL9544w2+/fZbAEJCQujSpQvNmjWjQIECHDp0iLlz5xIREcH3339PXFwc48ePvyEwnTx50g7KpUuX5s4776RBgwaULFmSmJgYNm/ezHfffUdsbCw///wzvXv35quvvvJasPF36QnMVhCeksr+RKCxiBQD5gO1M51BB3JNYAa466676N27N7fffvsN+x5++GE2b95Mv379iImJ4bfffmPhwoUpBsHp06dTo0YNr3/A//u//2Pq1Kl89NFHALz77rvcd9995M+fP9lxxYsX55577kkz34cPH7aDcuHChWnfvn2a5+SkBg0acPPNN1OvXj3q169PpUqVmDdvHq+++mq60qlRowa9evWifv361KtXj+rVq7NhwwZ69eqVqfw1bdqUO+64I1Np5AY//fSTHZTDwsKYMWPGDdV0ffr0oV+/fmzatIkVK1awcOFCunfvfkNaTZs25amnnqJNmzY39Fp44IEH6NOnD7179+avv/5i3759TJ06leeeey7rLi4LZcVcGcaYaBFZDbQAiolIsFVqrgScsg47BVQGTopIMFAUOO+x3c3znBTlmsa/F198kYkTJ6YYlN1uu+02nn/+eXt9wYIFKR5Xs2bNND9Vn3rqKW655RbAVVXyxx9/ZCDXLvPmzbOXO3Xq5PclkX79+vHiiy/SqVOnTFW7PPLIIwwePJguXbpw8803B0T/0uz0+eef28svvPBCim0nhQsX5r333iMkJASAcePG4aoa/Z9//vOffPnll7Rr185rV7KaNWsyfPhwe93b/05u4MNeGaWtkjIiUghoD+wBVgMPWof1AhZay4usdaz9q6x66kXAP6xeG9WBWsCG1F471wTmokWLOjquY8f/9UTZv39/pl6zRo0a9vK5c+cylIa7asUtpSoRpa6XlJTExo0bAVeg6dKli9djy5UrR3h4OACnT59m8+bNyfY7/d9p06YNoaGhdjqXL19O4wz/lC9fPsc/aSgPrBaR7cBGYLkxZgnwCjDQasQrCUyzjp8GlLS2DwQGARhjdgFfA7uBH4BnrCoSr3JVVYYTYWFh9nJsbGym0jpx4oS9XKpUqQyl8dNPP9lBvUaNGjRq1ChTeVJ5Q3R0tN0LpmTJkmkG12rVqvHzzz8DsHbtWpo1a5bu1wwKCqJgwYJ2ffS1a9coXLhwutPJaT7slbEdaJLC9sO4elVcv/0a8JCXtEYCI52+dsAF5gMHDtjLFSpUyHA6c+bMYefOnYDrH6Np06YZSsfzK6G/N/rlFuPHj2fIkCFEREQQEhJCqVKlaNSoEZ06daJdu3YBUWVyfXVEemT0m+L58+eJjIwEoFChQrm2Z0Yg/P0DLjC7G0vA9dUsLZs2beLChQsAxMXFcerUKdauXWvXKRcsWJCRI0fe0PDnRGRkJGvWrAEgODg41a+jyrlNmzbZy3FxcVy5coVjx46xaNEiGjVqxAcffEDFiqn2RvJ7RYsWJSQkhPj4eCIjI7l48SJFihTxevzRo0dTXE6Pr7/+2l5u1apVrp1wPrfm21NABeYtW7Ywf/58AAoUKMDjjz+e5jnvv/8+27dvv2F7UFAQ4eHhvPDCC9SpUydD+Vm8eDEJCa7ujm3atMlwdYhyKVCgAHfccQeNGzemUqVKhISE8Ndff7Fx40ZWrVpFYmIi27Zt49FHH+Wbb76hbNmyOZ3lDAsODqZhw4Zs3ryZpKQkFi9eTM+ePVM8NiIignXr/tdN9uLFi+l+vRMnTjBliqvXmIjQt2/fNM7wX1pi9iN//fUXL774ot13tn///pQrVy7D6VWoUIE777yT8uXLZzgNrcbwnY4dO/LYY49RrFixG/Y9/vjjHDhwgGeffZZjx45x9uxZBg8ezPTp03Mgp77z0EMP2Q15H374IY0aNaJ+/frJjrly5QovvfQS8fHx9rb0NtrFxMTw7LPPcvXqVQAeffTRNEfP+jMNzH4iJiaG/v37ExERAbj6O/+///f/HJ07e/bsZOkcPXqUH3/8kVmzZvHuu+/y+eefM27cOOrVq5euPO3cudOu6ytVqpSjahXlnbvroje1atVi6tSpdOnShdjYWH777Te2bduWqxtbu3TpwoIFC1i3bh2XL1/m0UcfpWvXrskGmMybN48zZ85QuXJlu7E6PV/lExMTeemll9i3bx8AdevW5ZVXXsmS68kugRCYHf8FRaSqiNxrLRcSkZuyLlvOxcbG8uyzz7Jjxw4AmjRpwnvvvZehP05oaCh169blhRdeYNasWYSFhfHnn3/y5JNPcvbs2XSl5dl3uUuXLgQHB8RnoF+rUqVKssEVa9euzcHcZF5QUBDjxo2zP9Tj4+OZO3cugwcPZuDAgXz88cecOXOG+vXrJ+uDnFpdtKekpCQGDx7MqlWrAKhevTpTp05NcVh3buLDSYxyjKPALCJP4ZqUY7K1qRLgtQe6iPQVkU0ismnq1KmZz6UXcXFxDBgwgPXr1wOuEWv//e9/7b6YmVGnTh369OkDuOrsPDv7pyU2NpalS5fa69p3Ofs0b/6/XkyHDx/OwZz4RpEiRZgyZQqTJk2iY8eOlCtXjvz581OkSBEaN27M0KFD+eqrr5IFmdQm3nIzxjBs2DAWLVoEuD7UZsyYERBD2QMhMDstxj2Dq9/eegBjzAERKePtYM/x5wkJCRnv95OK+Ph4Bg4caPfdrFOnDpMnT/Zpv8tWrVoxfvx4ALuzvxMrVqywG2AaN27MzTff7LM8qdR51kFnpBHMX7Vr14527dp53X/o0CF7+fp66JSMGDGCb775BnC1p8yYMSNXN5Z6CoSJ8p0G5lhjTJz7E8YaB54lAdeJhIQEXn75ZVavXg246h8/+eQTxyOcnPIcrHLp0iXH52mjX86Jjo62l51+pQ8EngWHtAaXvP3223z55ZeAa9TgzJkzM9Xn39/4c0nYKaeBea2IvAoUEpH2wP8Bi9M4J0skJiYyaNAge1rKGjVq8Mknn6TYWp9Zx48ft5edpn/69Gm761KhQoX429/+5vN8Ke88A1S1atVyLkXQ4FIAAB9FSURBVCPZKCoqyu4vX6RIkVQnyRozZgyfffYZ4KrymDFjBpUrV/Z6fG6UlwLzIFyz8+8Anga+Bz7Jqkx5k5SUxJAhQ+z62+rVqzNt2rQsqxdzf9UDV6OiEwsXLrS77HXo0CFZqVtlrePHjyf7tpJXesKMHj3aHr79z3/+0+skWR999JHdhbBUqVLMnDkzID+88tIAk+7AZ8aYrGvJS4MxhjfeeCNZY8X06dMdNXR4mj9/PmXKlOHOO+/0+skaFxfH2LFj7aqSkJAQHnjgAUd5XLhwob2ujX6+sWrVKuLi4mjfvr3X+kN3P2Z3gGrevHmGh9H7k61bt1K3bt0UR57GxcXx/vvv2/fczTffzL///e8U05k0aRL//e9/AShRogSffvppwLZ95KUScxfgQxH5CZgD/OAxg3+2GDt2LHPnzgVco6L+9a9/2V3kUnPnnXcmK0Hs3buXIUOGUK5cOVq0aMEtt9xCiRIlCAkJ4cKFC+zfv5+VK1cm6x738ssvU7169TRfa+PGjXZf0ipVqmRoIhl/cPLkyWRD2wG7nyvA+vXrSUxMPjlWhw4dqFu3brJtFy9evGGQx+nTp+3lPXv22HNeu4WHh9szpbmdOHGCUaNGUaJECVq3bk3t2rUpVaoUISEhnDt3jg0bNrBq1Sp7lGWZMmUcP7LK302aNImtW7fSpk0bGjRoQJkyZbh69SqHDh1i6dKlnDrlmta3bNmyTJw4McWubnPmzGHs2LH2es+ePTl27FiaDz647bbbKF68uG8vKBvkmcBsjOktIiHA34BHgY9FZLkx5skszZ2HrVu32ssJCQm8/fbbjs5btmxZivMm/Pnnn/bwbW9KlizJoEGD6Ny5s6PX8kwvNzf6nTp1yi5dpWTTpk3J5qsA1wdRSoE5tXT27duXLODD/4bCpyQyMpKFCxcm+1ZyvWbNmvHOO+8EVGPWhQsXWLx4cbLpYz3dcccdjBw50uvc2Vu2bEm27u5plJaZM2fmygcS5KWqDIwx8SKyFFdvjEK4qjeyLTD7yoABAwgPD2fjxo3s3r2bEydOEBUVRUJCAqGhoZQuXZpbb72V1q1b0759e8d9oq9cuWI3SObLl49u3bpl5WXkKV27dqVs2bJs2bKFXbt2cfbsWaKiorh69ar9uKpGjRrRuXPnXBlIUvPcc8/RoEEDNm7cyMmTJzl//jz58uWjdOnSNGnShM6dO+eZunSnAiEwi5PpBUXkb8AjQFtgDa5Jn5c5qc7Iqn7M6n8Cod+mUgDig3qIWbNmOY45//rXv/yy3sNpiflxXHXLT1uP4FZKKb+Ul+qYH83qjCillC8EfGAWkV+MMa1E5BLJR/oJYIwxeWdolVIqVwiEqr1UA7MxppX12y9mklNKqbQEQonZ6exyNUSkgLXcVkSecz/WWyml/EkgzC7ntF/JXCBRRGrimjWuMvBlluVKKaUyKBACs9NeGUnGmAQR6QGMN8aMF5EtaZ6llFLZLBD6MTsNzPEi8ijQC9fwbICQrMmSUkplnD+XhJ1yGph7A/2AkcaYIyJSHXD+SA+llMomeabEbIzZDTznsX4EeCerMqWUUhmVZwKziLQE3gCqWue4+zEH5ryBSqlcKy9VZUwDXgA2A4lpHKuUUjkmLwXmC8aYpWkfppRSOSsvBebVIvIuMA+wJzEyxvyRJblSSqkMykuB2T3JrecjOQxwt2+zo5RSmRPwc2W4GWPaZXVGlFLKFwKhxOx0royyIjLNeoIJIlJXRJ7I2qwppVT6+WpItohUFpHVIrJbRHaJyABrewkRWS4iB6zfxa3tIiLjROSgiGwXkaYeafWyjj8gIr3SuganHf5mAD8C7gep7Qeed3iuUkplm3z58jn+SUMC8KIxpi4QDjwjInWBQcBKY0wtYKW1Dq5notayfvoCk8AVyIFhuKqEmwPD3MHc6zU4vNZSxpivgSQA65FS2m1OKeV3fFViNsaccXdwMMZcAvYAFYFuwEzrsJm4nn+Ktf0z47IOKCYi5YGOwHJjTKQxJgpYDnRK7bWdBuYrIlISa7J8EQkHLjg8Vymlsk16ArOI9BWRTR4/fb2kWQ1oAqwHyhpjzli7/gTKWssVgRMep520tnnb7pXTXhkDgUVADRH5FSgNPOjwXKWUyjbpafwzxkzBNZVxaukVxjX18fPGmIue6RtjjIj4/IHTaT1a6iFjzDdAFHAXcCuu4dj7jDHxvs6MUkplli97ZYhICK6g/IUxZp61OUJEyhtjzlhVFWet7adwzVXvVsnadgpoe932Nam9blpVGYOt33ONMQnGmF3GmJ0alJVS/sqHvTIE13QUe4wxH3jsWoRrCmSs3ws9tj9u9c4IxzVi+gyujhMdRKS41ejXwdrmVVpVGZEisgy4WUQWXb/TGNM1jfOVUipb+bDE3BJ4DNghIlutba8Co4GvrS7Dx4CHrX3fA52Bg0AMrumSMcZEisgIYKN13HBjTGSq12CM9+oREckPNMU19/KT1+83xqxN68oSEhJ8Xv+ikguEkU5KgV1KzZRff/3Vccxp2bKlX45GSavEPM0Y85iITHUShJVSKqcFwsi/tALzbSJSAegpIlNxNfzZ0iqOAyQlJWUie8qJ4GCnnWtURsXFxeV0FvKE/PnzZzqNvBCY/4trZMvNuOZidhNcfZp1onyllF8J+MBsjBkHjBORSbiCdBtr10/GmG1ZnTmllEqvQAjMTkf+7QVmAaVwDS75XET6Z1mulFIqg3zVXS4nOa2cfAIIN8ZcARCRd4DfgfFZlTGllMqIPPMwVlx1yp6TFiVyXUOgUkr5A38uCTvlNDB/CqwXkfnWendcI2KUUkr5mNMnmHwgImuAVtam3saYLVmWK6WUyqC8VGJ2P3hVH76qlPJreSowK6VUbqCBWSml/Exe6pWhlFK5gpaYlVLKz2hgVkopPxMIgTn3V8YopVSA0RKzUiqgBEKJWQOzUiqgaK8MpZTyM1piVkopP6OBWSml/EwgBObcXxmjlFIBRkvMSqmAEgiNf7n/CpRSKsBoiVkpFVACoY451wRmYwxbt25l586d7Ny5kyNHjhAZGUl0dDQiQpEiRahVqxZt2rThvvvuo0iRIo7S/eWXX1iwYAHbt2/n/PnzFC5cmCpVqtC+fXsefPBBQkNDUz0/ISGBjRs3sm7dOnbu3Mnhw4e5cOECwcHBlCxZkvr169OpUyfatm1LUFCQL94Kv/fEE0/wyy+/2OujRo3i/vvvT/WcEydO8OWXX/L7779z6tQprl27RqlSpahbty5dunShU6dOWZ3tbOPLe3n37t1s376dXbt2ceDAAaKiooiKiiIxMZEiRYpQo0YNwsPD6d69O6VKlfKazsaNG+nTp0+6r6VChQr8+OOP6T4vKwVCYBZjTJa+QFxcnE9eIDY2lmbNmjk6tkSJEgwbNoy77747tXwxZMgQli5d6vWYypUr8+GHH3LrrbemuH/Dhg0MHDiQCxcupJmn+vXrM2bMGCpXrpz2BaRT/vz5fZ5mRs2fP59BgwYl25ZWYJ4yZQpjx44lISHB6zHh4eF89NFHFC9e3Gd5TY+4uDifpeXLe7lt27acP38+zXRCQ0N56aWXeOihh1Lcn9HA3Lx5c6ZN891T5vLnz5/pqHr06FHHMadatWp+GcVzTYnZrUyZMjRs2JBbbrmF8uXLExYWxrVr1zhy5AjLli3j2LFjREZGMnDgQCZOnMidd96ZYjqvvfYaP/zwAwDFihXjwQcfpFatWkRFRfHdd9+xY8cOTpw4wb///W++/PJLypUrd0MaERERdlAuWrQo4eHhNGzYkNKlS5OQkMCOHTtYvHgxly9fZufOnfTp04fZs2enWnLJzc6fP8/o0aMBVyCIiYlJ85yJEycyduxYwFXS6dChA61ataJw4cKcOHGC+fPnc+TIEdatW8fTTz/NZ599RsGCBbP0OrKLr+7l4sWL2+lUqlSJwoULEx8fz/Hjx1m9ejV79uwhJiaG4cOHExwcTI8ePW5Io2bNmnz00UeO8j1q1CgiIiIA6N69e8bfAOVVrikxJyUlceTIEWrUqOH1mMTEREaNGsWcOXMAqF69OosWLbrhuFWrVjFgwAAAypcvz8yZMylfvnyy1xo2bBgLFiwAoH379nzwwQc3pLN48WI+/fRTnnzySe69994US64RERH069ePgwcPAtC1a1dGjhyZjitPm7+UmJ9//nmWLl1K3bp1qVmzpv3eeysxHzhwgK5du5KUlERwcDATJkygXbt2yY6Jj4/npZdesj9E+/fvz7PPPpv1F3MdX5aYfXkvHzx4kBo1aqT69f2TTz6xP/xuuukm1qxZk+F75vDhw3Tr1g2AwoULs2rVKgoVKpShtFLiixLz8ePHHcecKlWq+GWJOdf0ysiXL1+qNzJAUFAQr7zyCsWKFQPgyJEjnDhx4objJk2aZC8PGTIkWVB2v9Zrr71mb1++fDkHDhy4IZ277rqLuXPn0rlzZ683etmyZRkzZoy9vmzZMq5evZrqdeRGK1euZOnSpeTLl4/hw4c7qk+fNWsWSUlJADz++OM3BGWAkJAQRo0aRYkSJQCYNm0aly9f9m3ms5kv7+WaNWumWaf65JNPcssttwBw6dIltmzJ+HOU3YUVgI4dO/o0KKv/yTWB2amQkBCqVKlir19f/3bs2DH27t0LQNWqVWnTpk2K6RQsWJAHHnjAXk+pgaNIkSKOGhpq1apF9erVAbh27RrHjx9P+0JykcuXL/Pmm28C0LNnTxo0aODovHXr1tnL7lJYSkJDQ+nQoQMAMTExrFixIhO5zT3SupfTw/OD4Ny5cxlKIzExkcWLF9vrKVWJ+AMRcfzjrwIuMCclJXH69Gl7vWTJksn2//rrr/aytzo7t1atWqV4XkaEhYXZy7GxsZlKy9+MGTOGiIgIypUrx/PPP+/4PHc9JWB/cHnjuf+nn35KfyZzobTu5fTwLG1ntI3j559/toN6jRo1aNSoUYbzk5U0MPsZYwzjx4+3b57atWvf0AvCXdcLULdu3VTTu/XWW+2v5IcOHSKj9fHx8fEcO3bMXq9QoUKG0vFHGzdu5OuvvwZg6NChFC5c2PG5GX0/9+/fn6HzchMn97JTX3/9NTt37gRcwb1JkyYZSsezGsOfG/18GZhFZLqInBWRnR7bSojIchE5YP0ubm0XERknIgdFZLuINPU4p5d1/AER6ZXW6+a6Xhluv/zyi13ydFcPrFy5kn379gGunhbur9eePANkxYoVU32N4OBgypQpw5kzZ7h69apdKkyvH374gUuXLgFQp06dgOmVERsby5AhQzDG0L59e+699950nV+qVClOnjwJuOpQa9eu7fXYI0eO2MtHjx7FGOPXJZ70yOi9fL1NmzbZvYTi4+M5deoUP/30E3/88Qfgqp4bMWJEhhr+IiMjWbt2LeD6v7jvvvvSnUYuNQOYAHzmsW0QsNIYM1pEBlnrrwB/A2pZP3cAk4A7RKQEMAxoBhhgs4gsMsZEeXvRXBuYhwwZkmKdW0hICG3btmXgwIFUqlTphv3uAAnYDSupKVasGGfOnLHPTW9gvnDhAh9++KG9/uSTT6brfH82YcIEjh49SlhYGEOHDk33+U2bNrUD88KFC70G5qtXr7J8+XJ7PT4+nqtXr6Y5+Ce3yOi9fL0PP/yQ7du337A9KCiI8PBwBgwYQJ06dTKUxyVLltj9zFu3bu3XhQtfzpVhjPlJRKpdt7kb0NZangmswRWYuwGfGddXwXUiUkxEylvHLjfGRAKIyHKgEzDb6zX47Ar8RPXq1QkPD7db8a/n2be2QIECaabnecyVK1fSlZfExET+85//8NdffwHQpk0buxErt9uzZw/Tp08H4IUXXqBs2bLpTuPhhx+2lz/77DO7ROYpPj6e11577YbAldt7ZjiR1r3sVPny5WnRosUNvY/Sw7Maw18b/dzSU5UhIn1FZJPHT18HL1HWGHPGWv4TcN/8FQHPrjMnrW3etnuVa0vMa9asAVx1cVeuXOHgwYMsWbKEb7/9lhEjRjB79mzGjRuXJSPtnBo9ejS//fYb4PrneOutt3IsL76UmJjIa6+9RkJCAg0aNKBnz54ZSuf222+nR48ezJ8/n4SEBPr160fHjh1p2bIlYWFhnDx5kvnz53P48GEqV66crAErEGYQc/PVvfzFF1/YyzExMRw9epRly5bxxRdf8N577/H5558zduxY6tWrl678uYd7g6v6qXXr1um7QD9mjJkCTMnE+UZEfD4YxNHdLSK3iMhKdwW4iDQUkSG+zkxGiAiFCxemcePGDBkyhI8//pigoCAOHjzIU089dcPoM8+vv056R3ge49mzIi3jxo3jq6++AlwNLlOmTMmx4cS+Nn36dHbt2kVwcDBvvfVWpoLk8OHD7RJYUlISS5cuZciQIbzwwgu8//77HD58mKpVq94wKs3pXCi5SXrv5dSEhoZSt25dnn/+eT777DPCwsKIiIjgqaee4uzZs+nK1/z58+3lLl26EBzs3+W5bOiVEWFVUWD9dr+hpwDPT89K1jZv271y+h81FRgMxAMYY7YD//B2sOfXg08++cThS/hGy5Yt7T6xp06dStbvElwjn9yio6PTTM/zGM9zUzN58mSmTp0KuIbLTp06lWrVqjk6198dO3aMCRMmANCrV69UG+ycyJ8/P6NHj+aLL76gW7duVKpUiYIFCxIWFkbdunUZOHAgCxYsoGjRovY5RYsW9ZvRjlkprXvZqTp16tC7d2/A1U4ya9Ysx+fGxsby/fff2+v+3BvDLRsC8yLA3bOiF7DQY/vjVu+McOCCVeXxI9BBRIpbPTg6WNu8cvrRF2qM2XDdhXidccbz64GvhmSnR8uWLZk3bx7g6s71yCOP2PuqVq3Khg0bANfNfvvtt3tNJyEhwS5dFCpUyFE96vTp0+3AVaRIEaZMmUKtWrUyfC3+ZvHixVy7dg0RITg4mIkTJ6Z4nLtHAcDq1av5888/AVff8IYNG95wfLNmzVKd2Mf9NwMcD2AJBKndy+nRqlUr+77ctGmT4/NWrlxpN5g3atSIm2++OUOvn5182VtHRGbjarwrJSIncfWuGA18LSJPAMcAd2PJ90Bn4CAQA/QGMMZEisgIYKN13HB3Q6A3TgPzORGpgaurByLyIHAm9VNyjmeVg2cvDHANYXXbvXt3qiWAffv2kZiYCJDmfATgasBy98C46aabmDx5cqZLlP7G3ffYGMPkyZMdnbNs2TKWLVsGuL5ipxSY0+IZmG+77bZ0n59bpXYvp4dnFV560slNjX5uvgzMxphHvey6J4VjDfCMl3SmA9Odvq7TwPwMrhJwbRE5BRwBMtbikw08hzxf3yWuZcuW9rK7Yc4bzzmFPc9LyezZs3n33XcB1z/TxIkTqV+/vuM8K+/i4uL47rvvAFfXr9wSIHwhtXs5PTwbTp2mc+bMGdavXw+4vjHmljmxA6F/u9PAbIwx94pIGJDPGHNJRFIfQ5tDkpKSkjVWNG7cONn+qlWrUqdOHfbs2cOxY8f4+eefU2xljo2NZe7cufZ6x44dvb7mt99+y6hRowDXDTxx4sQbXjdQ9O/fn/79+6d53KBBg+y/g5OJ8lPz3//+164K6dy5c6a6feUmad3L6fHNN9+kO52FCxfak0y1b98+XY3fKnOcNv7NBTDGXDHGuL8HfZs1WUrZ559/zrZt21I95sqVKwwePJg9e/YArkaiv/3tbzcc169fP3v5rbfesgeQuCUlJTFy5Eh7e/v27b3WEy9atIjhw4djjKFQoUJ8/PHHNG3aNMVj1Y127drltU9yUlIS06dPt+uxS5Qowauvvpqd2csSvrqXFyxYwG+//Zbq0Pb4+Hjee+89u0teSEiIow9JYwwLFy6013PTt5RAmCsj1RKziNQG6gFFRcTzr1kEyNbZyjdu3MiYMWOoWrUqzZs3p2bNmhQvXpx8+fIRFRXFnj17WLlypT0kNTg4mDfffDPFr2133303nTp14ocffuD06dM8/PDDPPTQQ9SqVYvo6GgWL17Mjh07AChdujT/+c9/UszTzz//zOuvv27/Y3Tv3p2LFy+ycuXKVK+lbt26eabUl5a5c+cyf/58WrduTePGjSlXrpw9t8iPP/5oz21y0003MWnSpEwPtvAHvrqX9+3bx9ChQylbtiwtWrTglltuoUSJEoSEhHDhwgX279/PqlWrknWPe/HFF9OcMApcDYTuUZlVqlRx/MQVf+DPAdeptKoybgXuA4oBXTy2XwKeyqpMpebYsWPJ5rtISaVKlXj99ddp0aKF12NGjhyJiLB06VKio6Pt7m2e3I+W8jYMe8eOHXbjILjqmWfP9jrK0jZixIhc0e0ou8TExPDjjz96fXZc3bp1efvttzM8nNhf+epejoiISNZIl5ISJUowaNCgFL9BpsSzCiW33asBH5iNMQuBhSLSwhjzezblKUUjRozg999/Z/Pmzezbt4+TJ08SHR2NMYbQ0FDKlStH7dq1adeuHXfddRchISGpppc/f37GjBlD165dmT9/Ptu3bycyMpKwsDCqVKlChw4dHD2MVWXOY489Rrly5diwYQPHjh3j3LlzGGPsB9l27NiRjh07BtSDbH11L/fv35877riDjRs3smfPHo4fP050dDQJCQmEhoZSqlQpateuTatWrbj33nsd38tXrlyx57zOly8fXbt29dm1Z4dACMyOHi0lIgWBJ3BVa9hVGMaYNJ/emBP9mPOavDDYIqf58tFSyjtfPFrqwoULjmNO0aJF/TKKO238+xwoB3QE1uIaUpjxTpVKKZVFAqHxz2lgrmmMGQpcMcbMBP6Oa75RpZRSPua0H3O89TtaROrjmuquTNZkSSmlMs6fS8JOOQ3MU6zJN4bgmqijMJD+mdGVUiqL5aXA/DnwAFAN14z98L/JoZVSSvmQ08C8ELgAbAYC6xHPSqmAkpdKzJWMMbljBhOllMrlnPbK+E1E8s4kuEqpXCsQusulNVfGDlxzMAcDvUXkMK6qDME141z6J9ZVSqks5M8B16m0qjLuy5ZcKKWUsqU1V0bqM6wopZSfCYQSc+A8A14ppQKEBmallPIzTrvLKaVUrhAIVRkamJVSASUQArNWZSillJ/RErNSKqBoiVkppZTPaYlZKRVQtMSslFLK57TErJQKKIFQYtbArJQKKIEQmLUqQyml/IyWmJVSAUVLzEoppXxOS8xKqYASCCVmMcbkdB78joj0NcZMyel8BDJ9j7Oevse5l1ZlpKxvTmcgD9D3OOvpe5xLaWBWSik/o4FZKaX8jAbmlGm9XNbT9zjr6XucS2njn1JK+RktMSullJ/RwKyUUn4mTwZmEXlDRF7K6XzkZiLynIjsEZEvRKSAiKwQka0i8ogPX+M3X6UVSESkmojsTMfx3UWkblbmSfmWjvxTGfV/wL3GmJMiEg5gjGnsyxcwxtzpy/TysO7AEmB3TmdEOZMnSswi8riIbBeRbSLy+XX7aojIDyKyWUR+FpHa1vYuIrJeRLZYpcGy1vY3RGS6iKwRkcMi8lxOXFN2EpGBIrLT+nleRP4L3AwsFZFXgFnA7VaJuYaI3CYia6339EcRKW+ls0ZE3hGRDSKyX0RaW9vrWdu2Wn+nWtb2y9bvr0Tk7x75mSEiD4pIkIi8KyIbrfOezu73JgcFichUEdklIstEpJCIPGW9F9tEZK6IhIrInUBX4F2Pv0+K97zyI8aYgP4B6gH7gVLWegngDeAla30lUMtavgNYZS0X53+9Vp4E3reW3wB+AwoApYDzQEhOX2cWvn+3ATuAMKAwsAtoAhz1eE/bAkus5RDr/SltrT8CTLeW13i8j52BFdbyeKCntZwfKGQtX7Z+9wBmeuw/ARTCNbJtiLW9ALAJqJ7T71k2/E2qAQlAY2v9a+BfQEmPY94C+lvLM4AHPfaleM/rj//85IWqjLuBb4wx5wCMMZHuSU5EpDBwJ/CNx8QnBazflYA5VmkvP3DEI83vjDGxQKyInAXKAiez+kJySCtgvjHmCoCIzANap3L8rUB9YLn1ngYBZzz2z7N+b8YVYAB+B14TkUrAPGPMgevSXAqMFZECQCfgJ2PMVRHpADQUkQet44oCtUj+twpUR4wxW61l93tZX0TeAorh+hD98fqT0rjnlZ/IC4E5NfmAaJNy3eh44ANjzCIRaYurpOwW67GciL6PngTYZYxp4WW/+72z3zdjzJcish74O/C9iDxtjFnlPsEYc01E1gAdcZXAv/J4rf7GmBsCUB5w/T1YCFfJuLsxZpuI/D9c32Sul9o9r/xEXqhjXgU8JCIlAUSkhHuHMeYicEREHrL2iYg0snYXBU5Zy72yMb/+5megu1VfGYarWuHnVI7fB5QWkRYAIhIiIvVSewERuRk4bIwZBywEGqZw2BygN67S+g/Wth+Bf4tIiJXOLVYe86qbgDPW+9HTY/sla19a97zyEwEfmI0xu4CRwFoR2QZ8cN0hPYEnrH27gG7W9jdwfd3bDJzLpuz6HWPMH7hKYhuA9cAnxpgtqRwfBzwIvGO9p1txfXVOzcPAThHZiqsa5LMUjlkG3IWrXjrO2vYJrp4Gf1jdxyaTt7+9DMX1N/oV2Oux/SvgZashuwbe73nlJ3RItlJK+ZmALzErpVRuo4FZKaX8jAZmpZTyMxqYlVLKz2hgVkopP6OBWSml/IwGZqWU8jP/HyDcN7tv7HusAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}