I have a multiprocess python service which I am exposing metrics from it. Here is the class:
import logging
from prometheus_client import multiprocess, CollectorRegistry, Counter, Histogram, Gauge, generate_latest
import os
logger = logging.getLogger(__name__)
prom_stats = os.environ["PROMETHEUS_MULTIPROC_DIR"]
if os.path.exists(prom_stats):
logger.debug(f"location: {prom_stats} exists")
else:
os.makedirs(prom_stats)
logger.debug(f"location: {prom_stats} created")
class Metrics:
def __init__(self):
logger.info("Init Metrics")
self.registry = CollectorRegistry()
multiprocess.MultiProcessCollector(self.registry)
self.metric = {}
#when label is defined, it get mandatory!
self.metric["PARSE_RATIO_GAUGE"] = Gauge('parse_ratio_gauge', 'some information', ['some_id'], registry=self.registry)
self.metric["PARSE_RATIO_GAUGE_NO_LABEL"] = Gauge('parse_ratio_gauge_2', 'some information', registry=self.registry)
def __getattribute__(self, __name: str):
return object.__getattribute__(self, __name)
def add_metric(self, metric, value, label=None):
if self.metric[metric]:
if self.metric[metric]._type == 'gauge':
if label:
self.metric[metric].labels(label).set(value)
else:
self.metric[metric].set(value)
elif self.metric[metric]._type == 'counter':
self.metric[metric].inc(value)
else:
logger.info(f"Unknown type for metric {metric} {self.metric[metric]}")
else:
logger.info(f"Unknown metric {metric}")
# export the metrics.
def export_metrics(self):
logger.info(self.registry)
return generate_latest(self.registry)
# Object initialization
if "METRICS_DEF" not in globals():
METRICS_DEF = Metrics()
The /metric endpoint shows this information:
# HELP parsed_total Number of parsed
# TYPE parsed_total counter
parsed_total 12.0
# HELP parse_ratio_gauge_2 some documentation
# TYPE parse_ratio_gauge_2 gauge
parse_ratio_gauge_2{pid="44497"} 4.326033333333333
parse_ratio_gauge_2{pid="109289"} 5.081816666666667
parse_ratio_gauge_2{pid="82672"} 5.135283333333334
parse_ratio_gauge_2{pid="102506"} 5.121183333333334
parse_ratio_gauge_2{pid="95982"} 5.523716666666667
parse_ratio_gauge_2{pid="48"} 9.760403225806451
parse_ratio_gauge_2{pid="37965"} 4.359083333333333
parse_ratio_gauge_2{pid="1"} 0.0
parse_ratio_gauge_2{pid="51282"} 4.33345
parse_ratio_gauge_2{pid="58065"} 12.358629032258065
parse_ratio_gauge_2{pid="24655"} 5.115733333333333
parse_ratio_gauge_2{pid="31182"} 4.728383333333333
parse_ratio_gauge_2{pid="89199"} 4.979333333333333
# HELP parse_ratio_gauge some documentation
# TYPE parse_ratio_gauge gauge
parse_ratio_gauge{pid="44497",some_id="09bdb"} 4.326033333333333
parse_ratio_gauge{pid="109289",some_id="09be3"} 5.081816666666667
parse_ratio_gauge{pid="82672",some_id="09bb9"} 5.135283333333334
parse_ratio_gauge{pid="102506",some_id="09bdb"} 5.121183333333334
parse_ratio_gauge{pid="95982",some_id="09bd3"} 5.523716666666667
parse_ratio_gauge{pid="48",some_id="991615"} 9.760403225806451
parse_ratio_gauge{pid="37965",some_id="09bd3"} 4.359083333333333
parse_ratio_gauge{pid="51282",some_id="09be3"} 4.33345
parse_ratio_gauge{pid="58065",some_id="991615"} 12.358629032258065
parse_ratio_gauge{pid="24655",some_id="09bb9"} 5.115733333333333
parse_ratio_gauge{pid="31182",some_id="09bca"} 4.728383333333333
parse_ratio_gauge{pid="89199",some_id="09bca"} 4.979333333333333
All is working fine. All I need to achieve, each time the application send a metric into the gauge, visualize in Grafana only one dot
I have 2 issues:
- The gauge has a PID label which for some reason, is interpreted in grafana as different - need to do abstraction of it
- Each time prometheus scrape the /metrics, it gets the same metric because it didn't change, and so grafan shows a constant line - which is wrong because the app didn't update the value in the gauge.
Two questions regarding this scenario:
- Is the gauge the right metric to use ? Our number is a float that can go up and down
- How to visualize it properly: one point in the graph at the first timestamp it was created/updated ? what i see what i want to see