...
Code Block |
---|
resource "grafana_notification_policy" "current-notifciation-policy" { group_by = ["alertname", "grafana_folder"] contact_point = grafana_contact_point.tfgrafana-test-slack-contact-pointdefault-email.name policy { matcher { label = "kubesend_to_serviceslack" match = "=" value = "advance-function-servicetrue" } group_by = ["..."] contact_point = grafana_contact_point.tf-test-emailslack-contact-point.name } } |
Run
terraform plan -auto-approve
...
For managing alert rules with terraform, follow these steps for each rule group. Make sure to use the grafana_data_source.KfuseDatasource.uid
and kfuse-tf-test-folder
in the alert rules. Here’s a working example of creating a test rulegroup with 1 rule in it. When an alert fires a notification is sent to the tf-test-slack-contact-point
because it is associated with the notification policy which matches the alert labels:
Code Block |
---|
resource "grafana_rule_group" "tf-test-alert-rulegroup" { name = "Test Alert Rules" folder_uid = grafana_folder.kfuse-tf-test-folder.uid interval_seconds = 60 org_id = 1 rule { name = "Test Alert" condition = "C" for = "0s" labels = { send_to_slack = true } // Query the datasource. data { ref_id = "A" relative_time_range { from = 600 to = 0 } datasource_uid = grafana_data_source.KfuseDatasource.uid // `model` is a JSON blob that sends datasource-specific data. // It's different for every datasource. The alert's query is defined here. model = jsonencode({ "expr": "avg(container_cpu_usage{container_name=~\".+\"})", "hide": false, "interval": "60s", "intervalMs": 15000, "maxDataPoints": 43200, "refId": "A" } ) } // The query was configured to obtain data from the last 60 seconds. Let's alert on the average value of that series using a Reduce stage. data { datasource_uid = "__expr__" // You can also create a rule in the UI, then GET that rule to obtain the JSON. // This can be helpful when using more complex reduce expressions. model = <<EOT {"conditions":[{"evaluator":{"params":[0,0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"params":[],"type":"last"},"type":"avg"}],"datasource":{"name":"Expression","type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"} EOT ref_id = "B" relative_time_range { from = 0 to = 0 } } // Now, let's use a math expression as our threshold. // We want to alert when the value of stage "B" above exceeds 70. data { datasource_uid = "__expr__" ref_id = "C" relative_time_range { from = 0 to = 0 } model = jsonencode({ expression = "$B > 70" type = "math" refId = "C" }) } } } |
...