DTS - 11 - Documentation & Misc Utilities.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**DTS - Complete Streets** *Right-of-Way Widths for Planned Street Improvements*\n",
    "\n",
    "\n",
    "# 11 - Documentation & Misc Utilities\n",
    "\n",
    "**Author:** rmangan\n",
    "\n",
    "____\n",
    "\n",
    "\n",
    "**Purpose:**\n",
    "\n",
    "Helper functions for generating data dictionaries and metadata.\n",
    "\n",
    "\n",
    "**This script performs the following functions:**\n",
    "\n",
    "1. Genereate data dictionaries for both internal and public modal composite datasets\n",
    "2. (potential) Create metadata and store in pandas dataframe\n",
    "3. (potential) Load metadata from pandas dataframe to feature class/feature service\n",
    "4. Drop extraneous fields from full modal composite dataset\n",
    "\n",
    "\n",
    "**Global Assumptions and Notes:**\n",
    "1. As no spatial calculations are performed here, analysis in this notebook could be performed without ArcPy and ArcGIS python modules by exporting the modal composite dataset to csv and reading it into a standard pandas dataframe.\n",
    "\n",
    "**Non-Standard Python Modules utilized:**\n",
    "1. arcpy 2.7 - required for reading file geodatabase data\n",
    "2. arcgis 1.83 - required for creating spatial data frame\n",
    "2. pandas 1.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import modules\n",
    "import arcpy\n",
    "import os\n",
    "import pandas as pd\n",
    "from arcgis.features import GeoAccessor, GeoSeriesAccessor\n",
    "from arcgis.gis import GIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define variables\n",
    "output_dd = r\"\\\\dc1vs01\\GISProj\\H\\Honolulu_DTS\\D3409300_RailActivation\\Tools\\GitHub\\DTS-Complete-Streets\\data_dictionary.csv\"\n",
    "\n",
    "output_gdb_path = r\"\\\\dc1vs01\\GISProj\\H\\Honolulu_DTS\\D3409300_RailActivation\\GeoData\\GDB\\scratch_GDBs\\modal_composite_output.gdb\"\n",
    "\n",
    "# Input Datasets\n",
    "modal_composite = r\"Z:\\H\\Honolulu_DTS\\D3409300_RailActivation\\GeoData\\GDB\\Modal\\Modal Composite final.gdb\\modal_composite_final\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Generate Data Dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get field objects\n",
    "fields = arcpy.ListFields(modal_composite)\n",
    "\n",
    "#empty lists for field properties\n",
    "name = []\n",
    "alias = []\n",
    "ftype = []\n",
    "length = []\n",
    "domain = []\n",
    "\n",
    "#iterate field objects and append properties to lists\n",
    "for field in fields:\n",
    "    name.append(field.name)\n",
    "    alias.append(field.aliasName)\n",
    "    ftype.append(field.type)\n",
    "    length.append(field.length)\n",
    "    domain.append(field.domain)\n",
    "\n",
    "\n",
    "#create dict with lists\n",
    "data_dict = {\n",
    "    \"Field_Name\": name,\n",
    "    \"Field_Alias\": alias,\n",
    "    \"Field_Type\": ftype,\n",
    "    \"Field_Length\": length,\n",
    "    \"Field_Domain\": domain\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Field_Name</th>\n",
       "      <th>Field_Alias</th>\n",
       "      <th>Field_Type</th>\n",
       "      <th>Field_Length</th>\n",
       "      <th>Field_Domain</th>\n",
       "      <th>Description</th>\n",
       "      <th>Source</th>\n",
       "      <th>Public</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>OBJECTID</td>\n",
       "      <td>OBJECTID</td>\n",
       "      <td>OID</td>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shape</td>\n",
       "      <td>Shape</td>\n",
       "      <td>Geometry</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SEGMENTID</td>\n",
       "      <td>SEGMENTID</td>\n",
       "      <td>Integer</td>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FULLNAME</td>\n",
       "      <td>FULLNAME</td>\n",
       "      <td>String</td>\n",
       "      <td>50</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NAME</td>\n",
       "      <td>NAME</td>\n",
       "      <td>String</td>\n",
       "      <td>40</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>row_width_future</td>\n",
       "      <td>ROW Width Future (greater of estimate &amp; 1986 p...</td>\n",
       "      <td>Single</td>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>ej_length</td>\n",
       "      <td>Economic Justice length (ft)</td>\n",
       "      <td>Double</td>\n",
       "      <td>8</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>ej_percent</td>\n",
       "      <td>Economic Justice %</td>\n",
       "      <td>Single</td>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>SLR_length</td>\n",
       "      <td>Sea Level Rise Exposure length (ft)</td>\n",
       "      <td>Double</td>\n",
       "      <td>8</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>SLR_percent</td>\n",
       "      <td>Sea Leve Rise Exposure %</td>\n",
       "      <td>Single</td>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>193 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Field_Name                                        Field_Alias  \\\n",
       "0            OBJECTID                                           OBJECTID   \n",
       "1               Shape                                              Shape   \n",
       "2           SEGMENTID                                          SEGMENTID   \n",
       "3            FULLNAME                                           FULLNAME   \n",
       "4                NAME                                               NAME   \n",
       "..                ...                                                ...   \n",
       "188  row_width_future  ROW Width Future (greater of estimate & 1986 p...   \n",
       "189         ej_length                       Economic Justice length (ft)   \n",
       "190        ej_percent                                 Economic Justice %   \n",
       "191        SLR_length                Sea Level Rise Exposure length (ft)   \n",
       "192       SLR_percent                           Sea Leve Rise Exposure %   \n",
       "\n",
       "    Field_Type  Field_Length Field_Domain Description Source Public  \n",
       "0          OID             4                                         \n",
       "1     Geometry             0                                         \n",
       "2      Integer             4                                         \n",
       "3       String            50                                         \n",
       "4       String            40                                         \n",
       "..         ...           ...          ...         ...    ...    ...  \n",
       "188     Single             4                                         \n",
       "189     Double             8                                         \n",
       "190     Single             4                                         \n",
       "191     Double             8                                         \n",
       "192     Single             4                                         \n",
       "\n",
       "[193 rows x 8 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#create df from dict\n",
    "df = pd.DataFrame.from_dict(data_dict)\n",
    "\n",
    "#add fields for further documentation (to be completed in excel)\n",
    "df[\"Description\"] = \"\"\n",
    "df[\"Source\"] = \"\"\n",
    "df[\"Public\"] = \"\"\n",
    "\n",
    "#print df for QC\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export df to csv\n",
    "df.to_csv(output_dd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Drop Extraneous fields\n",
    "\n",
    "Create a streamlined dataset of most commonly used fields. All fields should be retained in modal_composite_full."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop unecessary fields (all fields are avaialble in modal_composite_full dataset)\n",
    "\n",
    "#get all fields and add to list\n",
    "\n",
    "# fields = arcpy.ListFields(modal_composite)\n",
    "\n",
    "# for field in fields:\n",
    "#     field_list.append(field.name)\n",
    "    \n",
    "field_list = [i.name for i in arcpy.ListFields(modal_composite)]\n",
    "\n",
    "# for i in field_list:\n",
    "#     print(\"\\\"{}\\\",\".format(i))\n",
    "\n",
    "#fields to keep\n",
    "\n",
    "keep_fields = [\n",
    "    \"OBJECTID\",\n",
    "    \"Shape\",\n",
    "    \"SEGMENTID\",\n",
    "    \"FULLNAME\",\n",
    "    \"NAME\",\n",
    "    \"PROPERNAME\",\n",
    "    \"TYPE\",\n",
    "    \"POSTDIR\",\n",
    "    \"CLASS\",\n",
    "    \"STREET_CLA\",\n",
    "    \"ZIPCODER\",\n",
    "    \"ZIPCODEL\",\n",
    "    \"TOWNL\",\n",
    "    \"TOWNR\",\n",
    "    \"NEIGHBORL\",\n",
    "    \"NEIGHBORR\",\n",
    "    \"ONEWAY\",\n",
    "    \"OWNER\",\n",
    "    \"NBRightPed\",\n",
    "    \"SBLeftPedZ\",\n",
    "    \"NBRightP_1\",\n",
    "    \"SBLeftPe_1\",\n",
    "    \"NBRightP_2\",\n",
    "    \"SBLeftPe_2\",\n",
    "    \"NBRightP_3\",\n",
    "    \"SBLeftPe_3\",\n",
    "    \"NBRightP_4\",\n",
    "    \"SBLeftPe_4\",\n",
    "    \"NBRightP_5\",\n",
    "    \"SBLeftPedB\",\n",
    "    \"NBRightP_6\",\n",
    "    \"SBLeftPe_5\",\n",
    "    \"NBRightP_7\",\n",
    "    \"SBLeftPe_6\",\n",
    "    \"NBRightP_8\",\n",
    "    \"SBLeftPe_7\",\n",
    "    \"NBRightVis\",\n",
    "    \"SBLeftVisu\",\n",
    "    \"MidBlkXwkU\",\n",
    "    \"MidBlkXwkC\",\n",
    "    \"MidBlkXw_1\",\n",
    "    \"TraffCalmi\",\n",
    "    \"PPN_Final\",\n",
    "    \"HPI\",\n",
    "    \"PpnNotCity\",\n",
    "    \"FID_Ped_Improve_temp\",\n",
    "    \"Ped_Improve_ProjectID\",\n",
    "    \"Ped_Immprove_Extents\",\n",
    "    \"FID_Ped_Add_temp\",\n",
    "    \"Ped_Add_Project_ID\",\n",
    "    \"Fac_Name_BE\",\n",
    "    \"Fac_Desc_BE\",\n",
    "    \"Fac_Type_BE\",\n",
    "    \"Fac_Name_BP\",\n",
    "    \"Fac_Desc_BP\",\n",
    "    \"Fac_Name_BR\",\n",
    "    \"Fac_Desc_BR\",\n",
    "    \"Fac_Type_BR\",\n",
    "    \"TCAD_OID_2020\",\n",
    "    \"AB_SPEED\",\n",
    "    \"BA_SPEED\",\n",
    "    \"AB_CAPACIT\",\n",
    "    \"BA_CAPACIT\",\n",
    "    \"AB_LANE\",\n",
    "    \"BA_LANE\",\n",
    "    \"AB_CNT\",\n",
    "    \"BA_CNT\",\n",
    "    \"AB_FACTYPE\",\n",
    "    \"BA_FACTYPE\",\n",
    "    \"AB_FNCLASS\",\n",
    "    \"BA_FNCLASS\",\n",
    "    \"TOT_FLOW_D\",\n",
    "    \"max_row_2020\",\n",
    "    \"min_row_2020\",\n",
    "    \"mean_row_2020\",\n",
    "    \"f_class\",\n",
    "    \"fedaid_id\",\n",
    "    \"ln_exist\",\n",
    "    \"ln_prop\",\n",
    "    \"row_exist\",\n",
    "    \"row_prop\",\n",
    "    \"TCAD_OID_2040\",\n",
    "    \"ID\",\n",
    "    \"AB_VOC_MAX\",\n",
    "    \"BA_VOC_MAX\",\n",
    "    \"FACTYPE\",\n",
    "    \"ATYPE\",\n",
    "    \"TPN_OID\",\n",
    "    \"FULLNAME_1\",\n",
    "    \"Tier\",\n",
    "    \"NewTier\",\n",
    "    \"BE_OID_tmp\",\n",
    "    \"BP_OID_tmp\",\n",
    "    \"BR_OID_tmp\",\n",
    "    \"PED_ADD_OID_tmp\",\n",
    "    \"PED_IMPR_OID_tmp\",\n",
    "    \"Shape_Length\",\n",
    "    \"ln_exist_num\",\n",
    "    \"ln_prop_num\",\n",
    "    \"row_exist_num\",\n",
    "    \"row_prop_num\",\n",
    "    \"modal_width_ped_exist\",\n",
    "    \"modal_width_ped_prop\",\n",
    "    \"modal_width_bike_exist\",\n",
    "    \"modal_width_bike_prop\",\n",
    "    \"modal_width_auto_exist\",\n",
    "    \"modal_width_auto_prop\",\n",
    "    \"modal_width_bus_exist\",\n",
    "    \"modal_width_bus_prop\",\n",
    "    \"modal_width_park_exist\",\n",
    "    \"modal_width_park_prop\",\n",
    "    \"modal_width_medians\",\n",
    "    \"modal_width_max\",\n",
    "    \"Fac_Type_BP\",\n",
    "    \"lanes_assumed\",\n",
    "    \"VOC_max\",\n",
    "    \"median_type\",\n",
    "    \"median_width\",\n",
    "    \"progression\",\n",
    "    \"CS_example_num\",\n",
    "    \"CS_example_type\",\n",
    "    \"CS_type\",\n",
    "    \"length_urban_percent\",\n",
    "    \"length_urban\",\n",
    "    \"length_ag\",\n",
    "    \"length_ag_percent\",\n",
    "    \"length_ft\",\n",
    "    \"mean_row_area\",\n",
    "    \"modal_area_max\",\n",
    "    \"modal_area_diff\",\n",
    "    \"route_id\",\n",
    "    \"aadt\",\n",
    "    \"funsystem\",\n",
    "    \"funsysdesc\",\n",
    "    \"factype_1\",\n",
    "    \"factypedes\",\n",
    "    \"thrulanes\",\n",
    "    \"SPEED\",\n",
    "    \"SPD_LIMIT\",\n",
    "    \"MIN_SPD_LIMIT\",\n",
    "    \"MAX_SPD_LIMIT\",\n",
    "    \"LandValue\",\n",
    "    \"modal_width_ped_const\",\n",
    "    \"modal_width_bike_const\",\n",
    "    \"modal_width_auto_const\",\n",
    "    \"modal_width_bus_const\",\n",
    "    \"modal_width_park_const\",\n",
    "    \"modal_width_medians_const\",\n",
    "    \"modal_width_const\",\n",
    "    \"modal_area_const\",\n",
    "    \"modal_area_diff_const\",\n",
    "    \"bike_parallel_route\",\n",
    "    \"lane_count_rev\",\n",
    "    \"cs_type_revise\",\n",
    "    \"modal_width_auto_rule\",\n",
    "    \"modal_width_ped_furn\",\n",
    "    \"row_estimate_note\",\n",
    "    \"modal_area_auto_E\",\n",
    "    \"modal_area_auto_U\",\n",
    "    \"modal_area_auto_C\",\n",
    "    \"modal_area_ped_E\",\n",
    "    \"modal_area_ped_U\",\n",
    "    \"modal_area_ped_C\",\n",
    "    \"modal_area_bike_E\",\n",
    "    \"modal_area_bike_U\",\n",
    "    \"modal_area_bike_C\"\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get all fields and check against keep list to create drop list\n",
    "list_fields = arcpy.ListFields(modal_composite)\n",
    "fields = [i.name for i in list_fields]\n",
    "\n",
    "drop_list = []\n",
    "for i in fields:\n",
    "    if i in keep_fields:\n",
    "        print(\"{} field - keeping\".format(i))\n",
    "    else:\n",
    "        print(\"{} field - dropping\".format(i))\n",
    "        drop_list.append(i)\n",
    "\n",
    "#drop fields\n",
    "print(\"dropping fields..\")\n",
    "arcpy.DeleteField_management(modal_composite, drop_list)\n",
    "\n",
    "print(\"done\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}