How to prevent delay in chart rendering from .parquet data fetched from Flask backend?

Question

I am trying to create a simple GUI dashboard by fetching data using a back end Flask server by triggering an AJAX request when I interact with the multi check box drop down menus. Essentially, I have two drop down menus called "Select Date" and "Select Channel". The Select Date dropdown corresponds to a particular .parquet file and "Select Channel" corresponds to the column headers for those .parquet files. Thereafter, I am trying to render graphs as per the selected options.

Now, I am able to get the GUI to 'barely' function. When I say barely, I mean that everything works, except that it takes a lot of time to fetch and render the data(about 2 minutes for a single graph and 5 minutes for multiple graphs).

What I am not able to identify are the bottlenecks which are resulting in this latency. Inputs and suggestions to make the program more efficient would be very much appreciated. Perhaps there are problems with regard to how I am fetching the data or rendering the charts that isn't very memory efficient.

Here is my back end code:

from flask import Flask, render_template, json, url_for, request, jsonify
import os
import pandas as pd
import json
from flask_cors import CORS
from flask import Response
from cachetools import TTLCache

app = Flask(__name__)
CORS(app)
cache = TTLCache(maxsize=100, ttl=300)

# Function to get a list of available .parquet files
def get_parquet_files():
    SITE_ROOT = os.path.realpath(os.path.dirname(__file__))
    data_folder = os.path.join(SITE_ROOT, "static/data")
    parquet_files = [file for file in os.listdir(data_folder) if file.endswith('.parquet')]
    return parquet_files

# Route for the home page (index.html)
@app.route('/')
def showparquet():
    parquet_files = get_parquet_files()
    all_data = []
    SITE_ROOT = os.path.realpath(os.path.dirname(__file__))
    for index, parquet_file in enumerate(parquet_files):
        parquet_file_path = os.path.join(SITE_ROOT, "static/data", parquet_file)
        data = pd.read_parquet(parquet_file_path, engine='pyarrow')
        data['t'] = pd.to_datetime(data['t']).dt.tz_convert(None)
        a = data.columns.values.tolist()
        for column_name in a:
            if not column_name in all_data:
                all_data.append(column_name)


    return render_template('graphindex.html', parquet_files=parquet_files, data=all_data)


def fetch_data(channels, files):
    SITE_ROOT = os.path.realpath(os.path.dirname(__file__))
    all_data = []
    cache_key = (tuple(channels), tuple(files))
    cached_result = cache.get(cache_key)
    if cached_result:
        return cached_result

    for selected_file in files:
        parquet_file_path = os.path.join(SITE_ROOT, "static/data", selected_file)
        data = pd.read_parquet(parquet_file_path, engine='pyarrow')
        data['t'] = pd.to_datetime(data['t']).dt.tz_convert(None)
        data['t'] =data['t'].dt.strftime('%Y-%m-%dT%H:%M:%S.%fZ')

        selected_file_data = []
        for channel in channels:
            if channel in data.columns:
                channel_data = data[channel].tolist()
                for timestamp, value in zip(data['t'], channel_data):
                    record = {'timestamp': timestamp, 'channel': channel, 'value': value}
                    selected_file_data.append(record)

        # Filter out records with NaN values
        selected_file_data = [record for record in selected_file_data if not pd.isna(record['value'])]
        all_data.extend(selected_file_data)
    cache[cache_key] = all_data
    return all_data


# Route to handle the AJAX request and fetch data
@app.route('/fetch_data', methods=['POST'])
def handle_fetch_data():
    request_data = request.get_json()
    selected_channels = request_data.get('channels')
    selected_files = request_data.get('files')

    print(request_data)
    if not selected_channels or not selected_files:
        return jsonify(error="Invalid request. Please select both channels and files.")

    try:
        data = fetch_data(selected_channels, selected_files)
        print(data)
        return jsonify(data)
    except Exception as e:
        return jsonify(error=str(e))



if __name__ == '__main__':
    app.run(debug=True)

Following is the relevant front end code for fetching the data and rendering the charts dynamically based o n the options selected from the drop down menus:

  function fetchData(selectedChannels, selectedFiles) {
    console.log("Fetching data...");
    console.log(selectedChannels)
    console.log(selectedFiles)
    var graphno = selectedChannels.length * selectedFiles.length
    // Make an AJAX request to the backend with selectedChannel and selectedFile
    $.ajax({
      url: "/fetch_data",
      type: "POST",
      data: JSON.stringify({ channels: selectedChannels, files: selectedFiles }),
      contentType: "application/json",
      success: function (response) {
        console.log("Data received successfully:", response);
        // Render the data received from the backend on the webpage
        displayData(response);
      },
      error: function(xhr, status, error) {
        console.error("Error fetching data:", error);
      },
    });
  }

    

    // Function to generate random colors for the chart
   function getRandomColor() {
    const letters = '0123456789ABCDEF';
    let color = '#';
    for (let i = 0; i < 6; i++) {
      color += letters[Math.floor(Math.random() * 16)];
    }
    return color;
  }


 function displayData(data) {
  document.getElementById('chartContainer').innerHTML = '';
      // Prepare data for the charts
 
      const chartData = [];
      const chartOptions = {
      responsive: true,
      scales: {
    x: {
      type: 'time',
      time: {
        parser: 'luxon',
        unit: 'second',
        displayFormats: {
          second: 'HH:mm:ss',
          minute: 'HH:mm:ss',
          hour: 'HH:mm:ss',
          day: 'HH:mm:ss',
          week: 'HH:mm:ss',
          month: 'HH:mm:ss',
          quarter: 'HH:mm:ss',
          year: 'HH:mm:ss',
        },
        timezone: 'Europe/Paris'
      },
    },
        y: {
          beginAtZero: true,
        },

      },
      plugins: {
        legend: {
          display: true,
        },
        tooltip: {
        callbacks: {
          label: function (context) {
            var label = context.dataset.label || '';

            if (label) {
              label += ': ';
            }
            if (context.parsed.y !== null) {
              const timestamp = new Date(context.parsed.x).toUTCString();
               label += `${timestamp} - ${context.parsed.y}`;
              // label += context.parsed.y;
            }
            return label;
          },
        },
      },
        zoom: {
          pan: {
            enabled: true,
            mode: 'xy', // Enable both X and Y pan
            speed: 1,
            threshold: 0.1,
          },
          zoom: {
            wheel: {
              enabled: true,
            },
            pinch: {
              enabled: true,
            },
            mode: 'xy', // Enable both X and Y zoom
            
          },
        },
      },
    
    };

      // Create a map to store values for each channel and date combination
      const channelDateDataMap = new Map();

      // Loop through the data and aggregate values for each channel and date
      data.forEach((item) => {
        const { timestamp, value, channel } = item;

        // Convert the timestamp to a Luxon DateTime instance with the desired time zone (GMT)
        const dateTime = luxon.DateTime.fromISO(timestamp, { zone: 'GMT' });

        // Create a unique identifier for the graph using channel and date
        const graphId = `${channel}-${dateTime.toISODate()}`;

        // Check if the graph already exists in the channelDateDataMap
        if (channelDateDataMap.has(graphId)) {
          channelDateDataMap.get(graphId).push({ x: dateTime, y: value });
        } else {
          // If the graph is not in the map, create a new entry
          channelDateDataMap.set(graphId, [{ x: dateTime, y: value }]);
        }
      });

      // Clear the previous charts
      if (window.chartInstances) {
        Object.values(window.chartInstances).forEach((chart) => {
          chart.destroy();
        });
      }

      // Empty the chartInstances object
      window.chartInstances = {};

      // Loop through the channelDateDataMap to create datasets and chart instances
      channelDateDataMap.forEach((dataPoints, graphId) => {
        const randomColor = getRandomColor();
        chartData.push({
          label: graphId,
          data: dataPoints,
          borderColor: randomColor,
          backgroundColor: randomColor,
          pointRadius: 0.1,
          showLine: false,
        });

        const chartConfig = {
          type: 'scatter',
          data: {
            datasets: [chartData[chartData.length - 1]],
          },
          options: chartOptions,
        };
        const chartContainer = document.createElement('div');
        chartContainer.className = 'chart-container';

        const resetButton = document.createElement('button');
        resetButton.textContent = 'Reset Zoom';
        resetButton.onclick = () => resetZoom2(graphId);


        const ctx = document.createElement('canvas');
        ctx.width = 400;
        ctx.height = 200;
        chartContainer.appendChild(resetButton);
        chartContainer.appendChild(ctx);
        document.getElementById('chartContainer').appendChild(chartContainer);
        const newChartInstance = new Chart(ctx, chartConfig);

        // Store the chart instance in the window object
        window.chartInstances[graphId] = newChartInstance;
      });
    }

    function resetZoom2(graphId) {
  if (window.chartInstances && window.chartInstances[graphId]) {
    window.chartInstances[graphId].resetZoom();
  }
}

As for the event listeners for selecting the options from the drop down menus, they look like this:

$(function() {

  const msbChannel = document.getElementById("msbchannel");
  const msbDate = document.getElementById("msbdate");

  msbChannel.addEventListener("change", triggerDataFetch);
  msbDate.addEventListener("change", triggerDataFetch);

  
  function triggerDataFetch() {
  // const msbChannel = document.getElementById("msbchannel");
  // const msbDate = document.getElementById("msbdate");

  const selectedChannels = Array.from(msbChannel.querySelectorAll('.MultiCheckBoxDetail .cont input:checked')).map(input => input.value);
  const selectedFiles = Array.from(msbDate.querySelectorAll('.MultiCheckBoxDetail .cont input:checked')).map(input => input.value);
  const numSelectedChannels = selectedChannels.length;
  const numSelectedFiles = selectedFiles.length;

  if (numSelectedChannels >= 1 && numSelectedFiles >= 1 && numSelectedChannels <= 5 && numSelectedFiles <= 5) {
    console.log(selectedChannels);
    console.log(selectedFiles);
    fetchData(selectedChannels, selectedFiles);
  } else {
    console.log("Invalid selection");
  }
}

  });


  </script>

And this is the HTML code for creating the drop down menus:(The Third and the Fourth Dropdown menus are redundant for now and the charts/graphs are dynamically generated based on the selected options)

  <body>
  
    <div class="container">
      <div class="side-container">
        <!-- First Dropdown -->
      <div class="mb-3" data-dropdown-type="channel">
    <label for="exampleFormControlSelect1" class="form-label">Select Channel</label>
    <select id="test">
      {% for column in data %}
      <option value="{{ column }}">{{ column }}</option>
    {% endfor %}

    </select>

  </div>
  <!-- Second Dropdown -->

  <div class="mb-3" data-dropdown-type="date">
    <label for="exampleFormControlSelect1" class="form-label">Select Date</label>
    <select id="test_2">
      {% for parquet_file in parquet_files %}
      <option value="{{ parquet_file }}">{{ parquet_file }}</option>
    {% endfor %}
</select>
  </div>
  <!-- Third Dropdown -->
    <div class="mb-3">
    <label for="exampleFormControlSelect1" class="form-label">Start Time</label>
    <select id="test_3">
      <option value="" selected disabled>Select Start Time</option>
</select>

  </div>
<!-- Fourth Dropdown -->
      <div class="mb-3">
    <label for="exampleFormControlSelect1" class="form-label">End Time</label>
    <select id="test_4">
      <option value="" selected disabled>Select End Time</option>
</select>

  </div>
</div>

<!-- Graph to display data -->

    <div id="chartContainer">
</div>

<!-- Graph to display data -->
   
  </div>
</div>

I'm not sure if the following would help, but here's the JS for creating the dropdown menus:

        $(document).ready(function () {
            $(document).on("click", ".MultiCheckBox", function () {
                var detail = $(this).next();
                detail.show();
            });

            $(document).on("click", ".MultiCheckBoxDetailHeader input", function (e) {
                e.stopPropagation();
                var hc = $(this).prop("checked");
                $(this).closest(".MultiCheckBoxDetail").find(".MultiCheckBoxDetailBody input").prop("checked", hc);
                $(this).closest(".MultiCheckBoxDetail").next().UpdateSelect();
            });

            $(document).on("click", ".MultiCheckBoxDetailHeader", function (e) {
                var inp = $(this).find("input");    
                var chk = inp.prop("checked");
                inp.prop("checked", !chk);
                $(this).closest(".MultiCheckBoxDetail").find(".MultiCheckBoxDetailBody input").prop("checked", !chk);
                $(this).closest(".MultiCheckBoxDetail").next().UpdateSelect();
            });

            $(document).on("click", ".MultiCheckBoxDetail .cont input", function (e) {
                e.stopPropagation();
                $(this).closest(".MultiCheckBoxDetail").next().UpdateSelect();

                var val = ($(".MultiCheckBoxDetailBody input:checked").length == $(".MultiCheckBoxDetailBody input").length)
                $(".MultiCheckBoxDetailHeader input").prop("checked", val);
            });

            $(document).on("click", ".MultiCheckBoxDetail .cont", function (e) {
                var inp = $(this).find("input");
                var chk = inp.prop("checked");
                inp.prop("checked", !chk);

                var multiCheckBoxDetail = $(this).closest(".MultiCheckBoxDetail");
                var multiCheckBoxDetailBody = $(this).closest(".MultiCheckBoxDetailBody");
                multiCheckBoxDetail.next().UpdateSelect();

                var val = ($(".MultiCheckBoxDetailBody input:checked").length == $(".MultiCheckBoxDetailBody input").length)
                $(".MultiCheckBoxDetailHeader input").prop("checked", val);
            });

            $(document).mouseup(function (e) {
                var container = $(".MultiCheckBoxDetail");
                if (!container.is(e.target) && container.has(e.target).length === 0) {
                    container.hide();
                }
            });
        });

        var defaultMultiCheckBoxOption = { width: '220px', defaultText: 'Select Below', height: '200px', name:'' };

        jQuery.fn.extend({
            CreateMultiCheckBox: function (options) {
                    

                var localOption = {};
                localOption.width = (options != null && options.width != null && options.width != undefined) ? options.width : defaultMultiCheckBoxOption.width;
                localOption.defaultText = (options != null && options.defaultText != null && options.defaultText != undefined) ? options.defaultText : defaultMultiCheckBoxOption.defaultText;
                localOption.height = (options != null && options.height != null && options.height != undefined) ? options.height : defaultMultiCheckBoxOption.height;
                localOption.name = (options != null && options.name != null && options.name != undefined) ? options.name : defaultMultiCheckBoxOption.name;
                
                this.hide();
                this.attr("multiple", "multiple");
                var divSel = $("<div class='MultiCheckBox'>" + localOption.defaultText + "<span class='k-icon k-i-arrow-60-down'><svg aria-hidden='true' focusable='false' data-prefix='fas' data-icon='sort-down' role='img' xmlns='http://www.w3.org/2000/svg' viewBox='0 0 320 512' class='svg-inline--fa fa-sort-down fa-w-10 fa-2x'><path fill='currentColor' d='M41 288h238c21.4 0 32.1 25.9 17 41L177 448c-9.4 9.4-24.6 9.4-33.9 0L24 329c-15.1-15.1-4.4-41 17-41z' class=''></path></svg></span></div>").insertBefore(this);
                divSel.css({ "width": localOption.width });

                var detail = $("<div class='MultiCheckBoxDetail'><div class='MultiCheckBoxDetailHeader'><input type='checkbox' class='mulinput' value='-1982' /><div>Select All</div></div><div class='MultiCheckBoxDetailBody' id='" + localOption.name + "'></div></div>").insertAfter(divSel);
                detail.css({ "width": parseInt(options.width) + 10, "max-height": localOption.height });
                var multiCheckBoxDetailBody = detail.find(".MultiCheckBoxDetailBody");


                this.find("option").each(function () {
                    var val = $(this).attr("value");

                    if (val == undefined)
                        val = '';

                    multiCheckBoxDetailBody.append("<div class='cont'><div><input type='checkbox' class='mulinput' value='" + val + "' /></div><div>" + $(this).text() + "</div></div>");
                });

                multiCheckBoxDetailBody.css("max-height", (parseInt($(".MultiCheckBoxDetail").css("max-height")) - 28) + "px");

            },
            UpdateSelect: function () {
                var arr = [];

                this.prev().find(".mulinput:checked").each(function () {
                    arr.push($(this).val());
                });

                this.val(arr);
            },
        });