前言

日志监控解决方案中，常用EFK作为日志解决方案，我们需要针对业务日志进行告警，让开发人员及时收到消息。
Elastic Search的xpack有个alert组件，但是这个是收费的，经过研究发现elastalert还是不错的。

部署步骤

1. 安装ES，配置信任证书或者是账号密码
2. 配置smtp服务器和钉钉机器人
3. 配置all_any.yaml文件 自定义规则可参考elasalert官方文档

参考配置文件如下,本文件模板用于报警日志级别为ERROR的日志：

apiVersion: v1
data:
  all_any.yaml: >
    name:  prd log alert

    type: any

    index: project.xxxxx.*
 
    num_events: 1

    timeframe:
      minutes: 10
    filter:

    - query:
        query_string:
          query: "level: (emerg|alert|crit|err)"
    alert:

    - "dingtalk"

    email:

    - "xxxxxxx@qq.com"


    dingtalk_webhook_url:
    "https://oapi.dingtalk.com/robot/send?access_token=3f7ce1e337441b3dfbxxxx3e7" 

    dingtalk_webhook_secret:
    "SEC45d20d4c113xx5479dxxxx22aa79737354384606"

    dingtalk_msgtype: text


    realert:
      minutes: 1
    query_key:
      - kubernetes.namespace_name
      - kubernetes.labels.app
    exponential_realert:
      hours: 1
    aggregation:
      minutes: 1
    aggregate_by_match_time: true

    aggregation_key:
      - kubernetes.namespace_name
      - kubernetes.labels.app
    alert_subject: "Error {} @{}"

    alert_subject_args:
      - name
      - "@timestamp"

    alert_text_type: alert_text_only

    alert_text: |
      ### {}
      > Namespace: {}
      > App.Labels: {}
      > Pod: {}
      > Host: {}
      > Level: {}
      > Message: {}
    alert_text_args:
      - name
      - kubernetes.namespace_name
      - kubernetes.labels.app
      - kubernetes.pod_name
      - hostname
      - level
      - message
  config.json: |
    {
      "appName": "elastalert-server",
      "port": 3030,
      "wsport": 3333,
      "elastalertPath": "/opt/elastalert",
      "verbose": false,
      "es_debug": false,
      "debug": false,
      "rulesPath": {
        "relative": true,
        "path": "/rules"
      },
      "templatesPath": {
        "relative": true,
        "path": "/rule_templates"
      },
      "es_host": "es-service",
      "es_port": 9200,
      "writeback_index": "elastalert_status"
    }
  elastalert.yaml: |
    rules_folder: rules
    run_every:
      seconds: 60
    buffer_time:
      minutes: 15
    es_host: es-service
    es_port: 9200
    use_ssl: True
    verify_certs: True
    ca_certs: /opt/elastalert/certs/ca.crt
    client_cert: /opt/elastalert/certs/elastalert.crt
    client_key: /opt/elastalert/certs/elastalert.key
    writeback_index: elastalert_status
    writeback_alias: elastalert_alerts
    alert_time_limit:
      days: 2


    #邮箱告警可以配置在config中
    smtp_host: smtp.126.com
    smtp_port: 25
    #保存了邮箱验证的账号密码信息
    smtp_auth_file: /opt/elastalert/smtp_auth.yaml
    from_addr: xxxxxx@126.com
  smtp_auth.yaml: |
    #发送邮件的邮箱
    user: xxxxxx@126.com
    ##不是邮箱密码，是设置的POP3密码
    password: xxxxxxxx
kind: ConfigMap
metadata:
  name: elastalert-config

apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    deployment.kubernetes.io/revision: '1'
  labels:
    app: elastalert
  name: elastalert
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: elastalert
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: elastalert
      name: elastalert
    spec:
      containers:
        - env:
            - name: TZ
              value: Asia/Shanghai
          image: 'brycehuang/elastalert-dingtalk:3.0.0-beta.1'
          imagePullPolicy: IfNotPresent
          name: elastalert
          ports:
            - containerPort: 3030
              name: tcp-3030
              protocol: TCP
            - containerPort: 3333
              name: tcp-3333
              protocol: TCP
          resources:
            limits:
              cpu: '2'
            requests:
              cpu: '1'
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /opt/elastalert/config.yaml
              name: elastalert-config
              subPath: elastalert.yaml
            - mountPath: /opt/elastalert/smtp_auth.yaml
              name: elastalert-config
              subPath: smtp_auth.yaml
            - mountPath: /opt/elastalert-server/config/config.json
              name: elastalert-config
              subPath: config.json
            - mountPath: /opt/elastalert/rules/all_any.yaml
              name: elastalert-config
              subPath: all_any.yaml
            - mountPath: /opt/elastalert/certs
              name: elastalert-certs
              readOnly: true
            - mountPath: /opt/elastalert/server_data
              name: server-data
            - mountPath: /opt/logs
              name: elastalert-logs
            - mountPath: /opt/elastalert/rule_templates
              name: rule-templates
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      volumes:
        - configMap:
            defaultMode: 420
            name: elastalert-config
          name: elastalert-config
        - name: elastalert-certs
          secret:
            defaultMode: 420
            secretName: elastalert-secret
        - emptyDir: {}
          name: server-data
        - emptyDir: {}
          name: elastalert-logs
        - emptyDir: {}
          name: rule-templates