Using jsonnet to update Prometheus operator default rule

Question

I am trying to override the Default rules in Kube-Prometheus-Stack. I have enabled default rules when Kube-Prometheus-Stack was deployed. I tried with jsonnet as shown in this example https://github.com/prometheus-operator/kube-prometheus/blob/master/docs/developing-prometheus-rules-and-grafana-dashboards.md.

But I think I am not following the steps correctly so its not working as expected.

I followed the below steps..Kindly validate and please let me know what I am missing here.

Created a jsonnet file (my-custom-update.jsonnet) to update a rule "KubePodCrashLooping",

 local filter = {
  prometheusAlerts+:: {
    groups: std.map(
      function(group)
        if group.name == 'kubernetes-apps' then
          group {
            rules: std.filter(function(rule)
              rule.alert != "KubePodCrashLooping",
              group.rules
            )
          }
        else
          group,
      super.groups
    ),
  },
};
local update = {
  prometheusAlerts+:: {
    groups: std.map(
      function(group)
        if group.name == 'kubernetes-apps' then
          group {
            rules: std.map(
              function(rule)
                if rule.alert == "KubePodCrashLooping" then
                  rule {
                    expr: "kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\",statefulset!=\"vault\"} != kube_statefulset_status_replicas{job=\"kube-state-metrics\",statefulset!=\"vault\"}"
                  }
                else
                  rule,
                group.rules
            )
          }
        else
          group,
      super.groups
    ),
  },
};
local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + filter + update + {
    prometheusAlerts+:: (import 'prometheus-rules.json'),
};

{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } 

Build this jsonnet file with the build.sh script provided in the document,

./build.sh my-custom-update.jsonnet

This has actually deleted the "KubePodCrashLooping" rule instead of updating it. This created all the yaml files under manifests directory

I think I am missing something..Please advice.

Answer 1

I got it working now,

local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') +{
_config+:: {
namespace: 'monitoring',
},
prometheusAlerts+:: {
groups:
std.map(
function(group)
if group.name == 'kubernetes-apps' then
group {
rules: std.map(function(rule)
if rule.alert == "KubeStatefulSetReplicasMismatch" then
rule {
expr: "kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\",statefulset!=\"vault\"} != kube_statefulset_status_replicas{job=\"kube-state-metrics\",statefulset!=\"vault\"}",
labels: {
priority: 'P1',
severity: 'info',
},
}
else
rule
,
group.rules
)
}
else
group,
super.groups
),
},
};


{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }