Dispatch Plot improvements

src/DispatchPlot.py

@@ -5,11 +5,13 @@
   Date: 2021-05-18
 """
 import itertools as it
+from collections import defaultdict
+
 import matplotlib as mpl
 mpl.use('Agg') # Prevents the script from blocking while plotting
 import matplotlib.pyplot as plt
+
 from typing import List, Dict
-import random
 import numpy as np
 
 try:
@@ -21,6 +23,10 @@
   from ravenframework.PluginBaseClasses.OutStreamPlotPlugin import PlotPlugin, InputTypes, InputData
 
 
+# default color cycler, hatches
+colormap = plt.get_cmap('tab10').colors
+hatchmap = [None, '..', '\\\\', 'xx', '--', 'oo', '++', '**', 'OO']
+
 # Matplotlib Global Settings
 plt.rc("figure", figsize=(12, 8), titleweight='bold') # type: ignore
 plt.rc("axes", titleweight="bold", labelsize=12, axisbelow=True, grid=True) # type: ignore
@@ -106,70 +112,98 @@ def _group_by(iterable: List[str], idx: int) -> Dict[str, List[str]]:
         gr[key] = [var]
     return gr
 
-  def plot_component(self, fig, axes, df, grp_vars, comp_idx, sid, mstep, cid, cdict) -> None:
+  def plot_dispatch(self, figs, axes, df, grp_vars, sid, mstep, cid, cdict) -> None:
     """
       Plot and output the optimized dispatch for a specific sample, year, and cluster.
-      @ In, fig, matplotlib.figure.Figure, current figure used for plotting.
-      @ In, axes, List[List[matplotlib.Axes]], a list of axes to plot each variable.
+      @ In, figs, matplotlib.figure.Figure, current figures used for plotting.
+      @ In, axes, List[List[matplotlib.Axes]], a list of axes across figures to plot each variable.
       @ In, df, pandas.DataFrame, a dataframe containing data to plot.
       @ In, grp_vars, Dict[str, List[str]], a dictionary mapping components to variables.
-      @ In, comp_idx, Dict[str, int], a dictionary mapping components to numbers.
       @ In, sid, int, the sample ID.
       @ In, mstep, int, the macro step.
       @ In, cid, int, the cluster ID.
-      @ In, cdict, Dict[str, str], a dictionary contains color code to variables
+      @ In, cdict, Dict[Dict[str, Tuple[Float]]], a dictionary contains color code to variables
       @ Out, None
     """
-    # Pre-define color codes and transparency
-    Gray, Dark = ('#dcddde','#1a2b3c')
-    alpha = '70'
+    alpha = 0.7
+    time = df[self._microName].to_numpy()
     for (key, group), ax in zip(grp_vars.items(), axes.flat):
       # Define list for data, label, and color. Seperate 'level'(line plot) with other variables (stack plot)
       positive_dat = []
       positive_label = []
       positive_color = []
+      positive_hatch = []
+
       negative_dat = []
       negative_label = []
       negative_color = []
+      negative_hatch = []
+
       level_dat = []
       level_label = []
       level_color = []
 
-      # Secondary y axis for levels
+      # Secondary y axis for storage levels
       ax2 = ax.twinx()
+
       # Fill the lists
       for var in group:
         _, comp_name, tracker, _ = var.split('__')
-        comp_label = comp_name.replace('_', ' ').title()
+        comp_label = comp_name.replace('_', ' ')#.title()
         var_label = f'{comp_label}, {tracker.title()}'
         ls = '-'
         # Fill the positive, negative, and level lists
-        cindex = key + "," + comp_name # key for cdict dictionary
+        info = cdict[comp_name]#[res][tracker]
+        color = info['color']
+        hatch = info['hatch']
         if (df[var] != 0).any(): # no plotting variables that have all zeros values
           if tracker == 'level':
+            # this is the level of a storage component
             level_dat.append(var)
             level_label.append(var_label)
-            level_color.append(cdict.get(cindex))
+            level_color.append(color)
           else:
             if (df[var] > 0).any():
-                positive_dat.append(var)
-                positive_label.append(var_label)
-                positive_color.append(cdict.get(cindex))
+              # these are production tracking variables
+              positive_dat.append(var)
+              positive_label.append(var_label)
+              positive_color.append(tuple([*color, alpha]))
+              positive_hatch.append(hatch)
             else:
-                negative_dat.append(var)
-                negative_label.append(var_label)
-                negative_color.append(cdict.get(cindex))
+              # these are consumption tracking variables
+              negative_dat.append(var)
+              negative_label.append(var_label)
+              negative_color.append(tuple([*color, alpha]))
+              negative_hatch.append(hatch)
+
       # Plot the micro-step variable on the x-axis (i.e Time)
-      # Stackplot
-      if(len(positive_dat) > 0):
-        ax.stackplot(df[self._microName],*[df[key] for key in positive_dat],labels= positive_label, baseline='zero', colors= [color+alpha for color in positive_color[:len(negative_dat)]]+[Gray])
-      if(len(negative_dat) > 0):
-        ax.stackplot(df[self._microName],*[df[key] for key in negative_dat], labels= negative_label, baseline='zero', colors= [color+alpha for color in negative_color[:len(negative_dat)]] +[Gray])
-      # Lineplot
+      # center (0) line
+      ax.plot([time[0],time[-1]], [0,0], 'k-')
+      # Production
+      if len(positive_dat) > 0:
+        pos_stacks = ax.stackplot(time,
+                       *[df[key] for key in positive_dat],
+                       labels=positive_label,
+                       baseline='zero',
+                       colors=positive_color)
+        for stack, hatch in zip(pos_stacks, positive_hatch):
+          stack.set_hatch(hatch)
+
+      # Consumption
+      if len(negative_dat) > 0:
+        neg_stacks = ax.stackplot(time,
+                       *[df[key] for key in negative_dat],
+                       labels= negative_label,
+                       baseline='zero',
+                       colors=negative_color)
+        for stack, hatch in zip(neg_stacks, negative_hatch):
+          stack.set_hatch(hatch)
+
+      # Levels
       if(len(level_dat) > 0):
-        for key, c, llabel in zip(level_dat, level_color[:len(level_dat)] + [Dark], level_label[:len(level_dat)]):
+        for key, c, llabel in zip(level_dat, level_color, level_label):
           ax2.plot(df[self._microName], df[key], linestyle=ls, label=llabel, color=c)
-      # Sometimes users cases don't produce negative valued data so we need to check
+
       # Set figure title, legend, and grid
       ax.set_title(key.title().split('_')[-1])
       ax.set_xlabel(self._microName)
@@ -185,10 +219,12 @@ def plot_component(self, fig, axes, df, grp_vars, comp_idx, sid, mstep, cid, cdi
       ax.grid(None)
       ax2.grid(None)
     # Output and save the image
-    file_name = f"dispatch_id{sid}_y{mstep}_c{cid}.png"
-    fig.tight_layout()
-    fig.savefig(file_name)
-    self.raiseAMessage(f'Saved figure to "{file_name}"')
+    for f, fig in enumerate(figs):
+      file_name = f"dispatch_id{sid}_y{mstep}_c{cid}_f{f+1}.png"
+      fig.suptitle(f'Dispatch ID {sid} Year {mstep} Cluster {cid},\nFigure {f+1}/{len(figs)}')
+      fig.tight_layout()
+      fig.savefig(file_name)
+      self.raiseAMessage(f'Saved figure {f+1}/{len(figs)} to "{file_name}"')
     plt.clf()
 
   def plot_signal(self, fig, axes, df, sid, mstep, cid) -> None:
@@ -219,48 +255,41 @@ def plot_signal(self, fig, axes, df, sid, mstep, cid) -> None:
   def color_style(self, grp_vars):
     """
       @ In, grp_vars, Dict[str, List[str]], a dictionary mapping components to variables.
-      @ Out, colors, Dict[str, str], contains color code for variables
+      @ Out, comp_colors, Dict[Dict[str, Tuple[Float]]], contains rgb color code and hatching for components
     """
-    resources = [] # Determine the number of colormaps
-    technologis = [] # Determine the number of colors obtained from a colormap
-    for key, group in grp_vars.items():
-      resources.append(key)
-      for var in group:
-        _, comp_name, tracker, _ = var.split('__')
-        technologis.append(key + ',' + comp_name)
-    # remve duplicates
-    resources = list(dict.fromkeys(resources))
-    technologis = list(dict.fromkeys(technologis))
-    # colormap codes - can be changed to preferred colormaps - 17 in total 'Sequential' series
-    cm_codes = ['Purples', 'Blues', 'Greens', 'Oranges', 'Reds','YlOrBr', 'YlOrRd', 'OrRd', 'PuRd', 'RdPu', 'BuPu','GnBu', 'PuBu', 'YlGnBu', 'PuBuGn', 'BuGn', 'YlGn']
-    sample_cm = random.sample(cm_codes, len(resources))
-    resource_cm = {} # E.g. {'heat': 'OrRd', 'electricity': 'GnBu'} all string
-    i = 0
-    for s in resources:
-      resource_cm[s] = sample_cm[i] + '_r' #reverse colormap so it won't pick the lightest color that is almost invisible
-      i = i + 1
-    # Get the number of colors needed
-    resource_count = {} # E.g. {'heat': 5, 'electricity': 5}
-    for s in resources:
-      count = 0
-      for t in technologis:
-        if s in t:
-          count = count + 1
-      resource_count[s] = count
-    # Assign colors
-    colors = {}
-    for s in resources:
-      cm = mpl.cm.get_cmap(name= resource_cm[s])
-      # Get a subset of color map from 0 - 0.8 to avoid invisble light colors
-      cm = mpl.colors.LinearSegmentedColormap.from_list('trunc({n},{a:.2f},{b:.2f})'.format(n=cm.name, a=0, b=0.8),cm(np.linspace(0, 0.8)))
-      j = 0
-      for t in technologis:
-        clist = [cm(1.*i/resource_count[s]) for i in range(resource_count[s])] #color list
-        clist.reverse()
-        if s in t:
-          colors[t] = mpl.colors.rgb2hex(clist[j])
-          j = j + 1
-    return colors
+    # DESIGN:
+    #   -> components should be clearly different in color, and consistent across resource plots
+    # get the components, resources they use, and trackers per resource
+    # TODO this is just a rearrangement of the data, is it really useful, or is there another way?
+    comps = defaultdict(dict)
+    for res, group in grp_vars.items():
+      for variable in group:
+        _, comp, tracker, res = variable.split('__')
+        if res not in comps[comp]:
+          comps[comp][res] = [tracker]
+        else:
+          comps[comp][res].append(tracker)
+    n_comps = len(comps)
+    n_colors = len(colormap)
+    n_hatches = len(hatchmap)
+    n_uniques = n_colors * n_hatches
+    if n_comps > n_uniques:
+      self.raiseAWarning(f'A total of {n_comps} exist to plot, but only {n_uniques} unique identifying ' +
+                         'colors and patterns are available! This may lead to dispatch plot confusion.')
+    print('Assigning colors ...')
+    comp_colors = {}
+    # kept for easy debugging
+    #print('DEBUGG | c | ci | hi | comp | hatch | color_r | color_g | color_b')
+    for c, (comp, ress) in enumerate(comps.items()):
+      hatch_index, color_index = divmod(c, n_colors)
+      color = colormap[color_index]
+      hatch = hatchmap[hatch_index]
+      # kept for easy debugging
+      #print(f'DEBUGG | {c:2d} | {color_index:1d} | {hatch_index:1d} | {comp:20s} | '+
+      #      f'{hatch if hatch is not None else "None":4s} | '+
+      #      f'{color[0]*255:1.8f} | {color[1]*255:1.8f} | {color[2]*255:1.8f}')
+      comp_colors[comp] = {'color': color, 'hatch': hatch}
+    return comp_colors
 
   def run(self):
     """
@@ -276,14 +305,16 @@ def run(self):
     df = ds.to_dataframe().reset_index()
     dispatch_vars = list(filter(lambda x: "Dispatch__" in x, df.columns))
     grouped_vars = self._group_by(dispatch_vars, -1)
-    grouped_comp = self._group_by(dispatch_vars, 1)
-    comp_idx = {comp: i for i, comp in enumerate(grouped_comp.keys())}
+
     # Dimension variables to plot
     sample_ids = df[self._source.sampleTag].unique()
     cluster_ids = df['_ROM_Cluster'].unique()  # TODO: find way to not hardcode name
     macro_steps = df[self._macroName].unique()
+
     # Assign colors
     cdict = self.color_style(grouped_vars)
+
+    resources = set([x for x in grouped_vars])
     for sample_id, macro_step, cluster_id in it.product(sample_ids, macro_steps, cluster_ids):
       # Filter data to plot correct values for current dimension
       dat = df[
@@ -297,10 +328,22 @@ def run(self):
       # nature of the subplots, as well as the dynamic number of
       # components and signals to plot (i.e. dynamically nested subplots)
 
+      # If only 3 resources, make one figure; otherwise, 2 resources per figure
+      if len(resources) <= 3:
+        fig, res_axs = plt.subplots(len(resources), 1, sharex=True, squeeze=False)
+        res_figs = [fig]
+      else:
+        res_figs = []
+        res_axs = []
+        for _ in range(int(np.ceil(len(resources) / 2))):
+          fig, axs = plt.subplots(2, 1, sharex=True, squeeze=False)
+          res_figs.append(fig)
+          res_axs.extend(axs)
+
       # Output optimized component dispatch for current dimension.
-      fig0, axs0 = plt.subplots(len(grouped_vars), 1, sharex=True, squeeze=False)
-      self.plot_component(fig0, axs0, dat, grouped_vars, comp_idx, sample_id, macro_step, cluster_id, cdict)
+      res_axs = np.asarray(res_axs)
+      self.plot_dispatch(res_figs, res_axs, dat, grouped_vars, sample_id, macro_step, cluster_id, cdict)
 
       # Output synthetic time series signal for current dimension.
-      fig1, axs1 = plt.subplots(len(self._addSignals), 1, sharex=True, squeeze=False)
-      self.plot_signal(fig1, axs1, dat, sample_id, macro_step, cluster_id)
+      sig_figs, sig_axs = plt.subplots(len(self._addSignals), 1, sharex=True, squeeze=False)
+      self.plot_signal(sig_figs, sig_axs, dat, sample_id, macro_step, cluster_id)

tests/integration_tests/mechanics/debug_mode/tests

@@ -15,7 +15,7 @@
     [../]
     [./debug_plot]
       type = Exists
-      output = 'Debug_Run_o/dispatch_id0_y10_c0.png Debug_Run_o/dispatch_id0_y11_c0.png Debug_Run_o/dispatch_id1_y10_c0.png Debug_Run_o/dispatch_id1_y11_c0.png'
+      output = 'Debug_Run_o/dispatch_id0_y10_c0_f1.png Debug_Run_o/dispatch_id0_y11_c0_f1.png Debug_Run_o/dispatch_id1_y10_c0_f1.png Debug_Run_o/dispatch_id1_y11_c0_f1.png'
     [../]
     [./debug_plot]
       type = Exists

tests/integration_tests/mechanics/ramp_limits/tests

@@ -10,7 +10,7 @@
     [../]
     [./debug_plot]
       type = Exists
-      output = 'Sweep_Runs_o/dispatch_id0_y0_c0.png Sweep_Runs_o/dispatch_id0_y0_c0_SIGNAL.png'
+      output = 'Sweep_Runs_o/dispatch_id0_y0_c0_f1.png Sweep_Runs_o/dispatch_id0_y0_c0_SIGNAL.png'
     [../]
     [./debug_plot]
       type = Exists