Source code for jwql.website.apps.jwql.bokeh_dashboard

"""Bokeh based dashboard to monitor the status of the JWQL Application.
The dashboard tracks a variety of metrics including number of total
files per day, number of files per instrument, filesystem storage space,
etc.

The dashboard also includes a timestamp parameter. This allows users to
narrow metrics displayed by the dashboard to within a specific date
range.

Authors
-------

    - Mees B. Fix

Use
---

    The dashboard can be called from a python environment via the
    following import statements:
    ::

      from bokeh_dashboard impoer GeneralDashboard
      from monitor_template import secondary_function

Dependencies
------------

    The user must have a configuration file named ``config.json``
    placed in the ``jwql`` directory.
"""

from datetime import datetime as dt
from math import pi

from bokeh.models import Axis, ColumnDataSource, DatetimeTickFormatter, OpenURL, TapTool
from bokeh.models.widgets import Panel, Tabs
from bokeh.plotting import figure
from bokeh.transform import cumsum
import numpy as np
import pandas as pd

from jwql.utils.constants import FILTERS_PER_INSTRUMENT
from jwql.utils.utils import get_base_url
from jwql.website.apps.jwql.data_containers import build_table


[docs]def disable_scientific_notation(figure): """Disable y axis scientific notation. Parameters ---------- figure: bokeh figure object Returns ------- None """ yaxis = figure.select(dict(type=Axis, layout="left"))[0] yaxis.formatter.use_scientific = False
[docs]class GeneralDashboard: def __init__(self, delta_t=None): self.name = 'jwqldb_general_dashboard' self.delta_t = delta_t now = dt.now() self.date = pd.Timestamp('{}-{}-{}'.format(now.year, now.month, now.day))
[docs] def dashboard_filetype_bar_chart(self): """Build bar chart of files based off of type Parameters ---------- None Returns ------- tabs : bokeh.models.widgets.widget.Widget A figure with tabs for each instrument. """ # Make Pandas DF for filesystem_instrument # If time delta exists, filter data based on that. data = build_table('filesystem_instrument') if not pd.isnull(self.delta_t): data = data[(data['date'] >= (self.date - self.delta_t)) & (data['date'] <= self.date)] # Set title and figures list to make panels title = 'File Types per Instrument' figures = [] # Group by instrument/filetype and sum the number of files that have that specific combination data_by_filetype = data.groupby(["instrument", "filetype"]).size().reset_index(name="count") # For unique instrument values, loop through data # Find all entries for instrument/filetype combo # Make figure and append it to list. for instrument in data.instrument.unique(): index = data_by_filetype["instrument"] == instrument figures.append(self.make_panel(data_by_filetype['filetype'][index], data_by_filetype['count'][index], instrument, title, 'File Type')) tabs = Tabs(tabs=figures) return tabs
[docs] def dashboard_instrument_pie_chart(self): """Create piechart showing number of files per instrument Parameters ---------- None Returns ------- plot : bokeh.plotting.figure Pie chart figure """ # Replace with jwql.website.apps.jwql.data_containers.build_table data = build_table('filesystem_instrument') if not pd.isnull(self.delta_t): data = data[(data['date'] >= self.date - self.delta_t) & (data['date'] <= self.date)] try: file_counts = {'nircam': data.instrument.str.count('nircam').sum(), 'nirspec': data.instrument.str.count('nirspec').sum(), 'niriss': data.instrument.str.count('niriss').sum(), 'miri': data.instrument.str.count('miri').sum(), 'fgs': data.instrument.str.count('fgs').sum()} except AttributeError: file_counts = {'nircam': 0, 'nirspec': 0, 'niriss': 0, 'miri': 0, 'fgs': 0} data = pd.Series(file_counts).reset_index(name='value').rename(columns={'index': 'instrument'}) data['angle'] = data['value'] / data['value'].sum() * 2 * pi data['color'] = ['#F8B195', '#F67280', '#C06C84', '#6C5B7B', '#355C7D'] plot = figure(title="Number of Files Per Instruments", toolbar_location=None, tools="hover,tap", tooltips="@instrument: @value", x_range=(-0.5, 1.0)) plot.wedge(x=0, y=1, radius=0.4, start_angle=cumsum('angle', include_zero=True), end_angle=cumsum('angle'), line_color="white", color='color', legend='instrument', source=data) url = "{}/@instrument".format(get_base_url()) taptool = plot.select(type=TapTool) taptool.callback = OpenURL(url=url) plot.axis.axis_label = None plot.axis.visible = False plot.grid.grid_line_color = None return plot
[docs] def dashboard_files_per_day(self): """Scatter of number of files per day added to ``JWQLDB`` Parameters ---------- None Returns ------- tabs : bokeh.models.widgets.widget.Widget A figure with tabs for each instrument. """ source = build_table('filesystem_general') if not pd.isnull(self.delta_t): source = source[(source['date'] >= self.date - self.delta_t) & (source['date'] <= self.date)] date_times = [pd.to_datetime(datetime).date() for datetime in source['date'].values] source['datestr'] = [date_time.strftime("%Y-%m-%d") for date_time in date_times] p1 = figure(title="Number of Files Added by Day", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=1700, x_axis_label='Date', y_axis_label='Number of Files Added') p1.line(x='date', y='total_file_count', source=source, color='#6C5B7B', line_dash='dashed', line_width=3) disable_scientific_notation(p1) tab1 = Panel(child=p1, title='Files Per Day') p2 = figure(title="Available & Used Storage", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=1700, x_axis_label='Date', y_axis_label='Storage Space [Terabytes?]') p2.line(x='date', y='available', source=source, color='#F8B195', line_dash='dashed', line_width=3, legend='Available Storage') p2.line(x='date', y='used', source=source, color='#355C7D', line_dash='dashed', line_width=3, legend='Used Storage') disable_scientific_notation(p2) tab2 = Panel(child=p2, title='Storage') p1.xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"], days=["%d %B %Y"], months=["%d %B %Y"], years=["%d %B %Y"], ) p1.xaxis.major_label_orientation = pi / 4 p2.xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"], days=["%d %B %Y"], months=["%d %B %Y"], years=["%d %B %Y"], ) p2.xaxis.major_label_orientation = pi / 4 tabs = Tabs(tabs=[tab1, tab2]) return tabs
[docs] def dashboard_monitor_tracking(self): """Build bokeh table to show status and when monitors were run. Parameters ---------- None Returns ------- table_columns : numpy.ndarray Numpy array of column names from monitor table. table_values : numpy.ndarray Numpy array of column values from monitor table. """ data = build_table('monitor') if not pd.isnull(self.delta_t): data = data[(data['start_time'] >= self.date - self.delta_t) & (data['start_time'] <= self.date)] data['start_time'] = data['start_time'].map(lambda x: x.strftime('%m-%d-%Y %H:%M:%S')) data['end_time'] = data['end_time'].map(lambda x: x.strftime('%m-%d-%Y %H:%M:%S')) # data = data.drop(columns='affected_tables') table_values = data.sort_values(by='start_time', ascending=False).values table_columns = data.columns.values return table_columns, table_values
[docs] def make_panel(self, x_value, top, instrument, title, x_axis_label): """Make tab panel for tablulated figure. Parameters ---------- x_value : str Name of value for bar chart. top : int Sum associated with x_label instrument : str Title for the tab title : str Figure title x_axis_label : str Name of the x axis. Returns ------- tab : bokeh.models.widgets.widget.Widget Return single instrument panel """ # filetypes = data.filetype.unique() data = pd.Series(dict(zip(x_value, top))).reset_index(name='top').rename(columns={'index': 'x'}) source = ColumnDataSource(data) plot = figure(x_range=x_value, title=title, plot_width=850, tools="hover", tooltips="@x: @top", x_axis_label=x_axis_label) plot.vbar(x='x', top='top', source=source, width=0.9, color='#6C5B7B') plot.xaxis.major_label_orientation = pi / 4 disable_scientific_notation(plot) tab = Panel(child=plot, title=instrument) return tab
[docs] def dashboard_exposure_count_by_filter(self): """Create figure for number of files per filter for each JWST instrument. Parameters ---------- None Returns ------- tabs : bokeh.models.widgets.widget.Widget A figure with tabs for each instrument. """ # for instrument in data.instrument.unique(): title = 'File Counts Per Filter' figures = [self.make_panel(FILTERS_PER_INSTRUMENT[instrument], np.random.rand(len(FILTERS_PER_INSTRUMENT[instrument])) * 10e7, instrument, title, 'Filters') for instrument in FILTERS_PER_INSTRUMENT] tabs = Tabs(tabs=figures) return tabs
[docs] def dashboard_anomaly_per_instrument(self): """Create figure for number of anamolies for each JWST instrument. Parameters ---------- None Returns ------- tabs : bokeh.models.widgets.widget.Widget A figure with tabs for each instrument. """ from jwql.utils.constants import ANOMALY_CHOICES_PER_INSTRUMENT # Set title and figures list to make panels title = 'Anamoly Types per Instrument' figures = [] # For unique instrument values, loop through data # Find all entries for instrument/filetype combo # Make figure and append it to list. for instrument in ANOMALY_CHOICES_PER_INSTRUMENT.keys(): data = build_table('{}_anomaly'.format(instrument)) data = data.drop(columns=['id', 'rootname', 'user']) if not pd.isnull(self.delta_t) and not data.empty: data = data[(data['flag_date'] >= (self.date - self.delta_t)) & (data['flag_date'] <= self.date)] summed_anamoly_columns = data.sum(axis=0).to_frame(name='counts') figures.append(self.make_panel(summed_anamoly_columns.index.values, summed_anamoly_columns['counts'], instrument, title, 'Anomaly Type')) tabs = Tabs(tabs=figures) return tabs