IMS系统监控与告警设计完全指南

监控与告警是企业级 IMS 系统稳定运行的核心保障。一个完善的监控告警体系需要覆盖基础设施、应用服务、业务指标等多个维度，及时发现并处理问题，保障系统 7x24 小时稳定运行。

监控体系架构

监控系统采用分层架构设计：

• 指标采集层 - 收集各类监控指标数据
• 数据存储层 - 存储时序数据，如 Prometheus
• 告警引擎层 - 规则匹配、告警判定
• 通知服务层 - 多渠道告警通知
• 可视化层 - 监控面板与报表

指标采集设计

监控指标分为基础设施指标、应用指标和业务指标三类：

// 指标定义与采集
class MetricsCollector {
  constructor() {
    this.metrics = new Map();
  }

  // 记录计数器指标
  incrementCounter(name, labels = {}) {
    const key = this.getKey(name, labels);
    const metric = this.metrics.get(key) || { type: 'counter', value: 0, labels };
    metric.value += 1;
    this.metrics.set(key, metric);
  }

  // 记录 Gauge 指标
  setGauge(name, value, labels = {}) {
    const key = this.getKey(name, labels);
    this.metrics.set(key, { type: 'gauge', value, labels });
  }

  // 记录直方图指标
  observeHistogram(name, value, labels = {}) {
    const key = this.getKey(name, labels);
    let metric = this.metrics.get(key);
    if (!metric) {
      metric = { type: 'histogram', values: [], labels };
      // 定义直方图 buckets
      metric.buckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10];
      metric.count = 0;
      metric.sum = 0;
      this.metrics.set(key, metric);
    }
    metric.values.push(value);
    metric.count += 1;
    metric.sum += value;
  }

  getKey(name, labels) {
    const labelStr = Object.entries(labels).map([k, v] => `${k}=${v}`).join(',');
    return `${name}{${labelStr}}`;
  }
}

核心监控指标

IMS 系统需要监控的核心指标：

// IMS 系统核心监控指标
const IMSEvents = {
  // HTTP 请求指标
  httpRequest: (req, res, duration) => {
    collector.incrementCounter('ims_http_requests_total', {
      method: req.method,
      path: req.route.path,
      status: res.statusCode
    });
    collector.observeHistogram('ims_http_request_duration_seconds', duration, {
      method: req.method,
      path: req.route.path
    });
  },

  // 数据库操作指标
  dbQuery: (operation, table, duration, success) => {
    collector.incrementCounter('ims_db_queries_total', {
      operation,
      table,
      status: success ? 'success' : 'error'
    });
    collector.observeHistogram('ims_db_query_duration_seconds', duration, {
      operation,
      table
    });
  },

  // API 接口性能指标
  apiCall: (apiName, duration, success) => {
    collector.incrementCounter('ims_api_calls_total', {
      api: apiName,
      status: success ? 'success' : 'error'
    });
    collector.observeHistogram('ims_api_call_duration_seconds', duration, {
      api: apiName
    });
  },

  // 用户会话指标
  userSession: (event, count = 1) => {
    collector.incrementCounter('ims_user_sessions_total', { event: event }, count);
    collector.setGauge('ims_active_sessions', sessionManager.activeCount);
  },

  // 工作流指标
  workflow: (processDefId, event, duration) => {
    collector.incrementCounter('ims_workflow_events_total', {
      process: processDefId,
      event: event
    });
    if (duration) {
      collector.observeHistogram('ims_workflow_duration_seconds', duration, {
        process: processDefId,
        event: event
      });
    }
  },

  // 消息队列指标
  mqMessage: (topic, action, success) => {
    collector.incrementCounter('ims_mq_messages_total', {
      topic,
      action: action,
      status: success ? 'success' : 'error'
    });
  },

  // 缓存指标
  cache: (operation, key, hit) => {
    collector.incrementCounter('ims_cache_operations_total', {
      operation,
      result: hit ? 'hit' : 'miss'
    });
  },

  // 业务指标 - 表单提交
  formSubmit: (formId, success) => {
    collector.incrementCounter('ims_form_submits_total', {
      form: formId,
      status: success ? 'success' : 'error'
    });
  }
};

告警规则设计

告警规则支持阈值告警、趋势告警和复合条件告警：

// 告警规则定义
const AlertRules = [
  {
    id: 'high_error_rate',
    name: '错误率过高',
    expression: 'rate(ims_http_requests_total{status=~"5.."}[5m]) > 0.05',
    severity: 'critical',
    for: '2m',
    labels: { team: 'backend' },
    annotations: {
      summary: 'HTTP 错误率超过 5%',
      description: '当前错误率: {{ $value }}'
    }
  },
  {
    id: 'high_response_time',
    name: '响应时间过长',
    expression: 'histogram_quantile(0.95, rate(ims_http_request_duration_seconds_bucket[5m])) > 2',
    severity: 'warning',
    for: '5m',
    labels: { team: 'backend' },
    annotations: {
      summary: 'P95 响应时间超过 2 秒'
    }
  },
  {
    id: 'db_connection_pool_exhausted',
    name: '数据库连接池耗尽',
    expression: 'ims_db_connections_active / ims_db_connections_max > 0.9',
    severity: 'critical',
    for: '1m',
    labels: { team: 'dba' },
    annotations: {
      summary: '数据库连接池使用率超过 90%'
    }
  },
  {
    id: 'disk_space_low',
    name: '磁盘空间不足',
    expression: 'node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} < 0.1',
    severity: 'critical',
    for: '5m',
    labels: { team: 'ops' },
    annotations: {
      summary: '磁盘空间剩余不足 10%'
    }
  },
  {
    id: 'memory_leak_suspect',
    name: '内存泄漏嫌疑',
    expression: 'increase(ims_process_resident_memory_bytes[30m]) > 1073741824',
    severity: 'warning',
    for: '10m',
    labels: { team: 'backend' },
    annotations: {
      summary: '30分钟内内存增长超过 1GB，可能存在内存泄漏'
    }
  },
  {
    id: 'queue_backlog',
    name: '消息队列积压',
    expression: 'ims_mq_queue_depth > 10000',
    severity: 'warning',
    for: '5m',
    labels: { team: 'backend' },
    annotations: {
      summary: '消息队列积压超过 10000 条'
    }
  }
];

// 告警规则引擎
class AlertEngine {
  constructor(rules) {
    this.rules = rules;
    this.activeAlerts = new Map();
    this.checkInterval = 10000;  // 10秒检查一次
  }

  async start() {
    this.timer = setInterval(() => this.checkAllRules(), this.checkInterval);
  }

  async checkAllRules() {
    for (const rule of this.rules) {
      await this.evaluateRule(rule);
    }
  }

  async evaluateRule(rule) {
    const queryResult = await prometheus.query(rule.expression);
    const isFiring = queryResult.length > 0;

    if (isFiring) {
      if (!this.activeAlerts.has(rule.id)) {
        // 新告警
        await this.triggerAlert(rule, queryResult);
      } else {
        // 告警持续，检查是否需要升级
        const existingAlert = this.activeAlerts.get(rule.id);
        await this.checkAlertEscalation(existingAlert, queryResult);
      }
    } else {
      // 告警恢复
      if (this.activeAlerts.has(rule.id)) {
        await this.resolveAlert(rule.id);
      }
    }
  }

  async triggerAlert(rule, values) {
    const alert = {
      id: rule.id,
      name: rule.name,
      severity: rule.severity,
      labels: rule.labels,
      annotations: this.interpolateAnnotations(rule.annotations, values[0]),
      startsAt: new Date(),
      fingerprint: generateFingerprint(rule)
    };

    this.activeAlerts.set(rule.id, alert);

    // 发送告警通知
    await AlertNotifier.notify('firing', alert);
  }

  async resolveAlert(alertId) {
    const alert = this.activeAlerts.get(alertId);
    alert.endsAt = new Date();
    alert.status = 'resolved';

    // 发送恢复通知
    await AlertNotifier.notify('resolved', alert);

    this.activeAlerts.delete(alertId);
  }
}

告警通知设计

支持多渠道告警通知，包括邮件、短信、钉钉、企业微信等：

class AlertNotifier {
  constructor() {
    this.channels = new Map();
    this.registerChannels();
  }

  registerChannels() {
    // 邮件通知
    this.channels.set('email', new EmailNotifier());

    // 钉钉通知
    this.channels.set('dingtalk', new DingTalkNotifier());

    // 企业微信通知
    this.channels.set('wechat', new WeChatNotifier());

    // 短信通知（严重告警）
    this.channels.set('sms', new SMSNotifier());

    // 电话通知（严重告警）
    this.channels.set('phone', new PhoneNotifier());
  }

  async notify(status, alert) {
    // 根据告警级别选择通知渠道
    const channels = this.getChannelsForAlert(alert);

    for (const channel of channels) {
      const notifier = this.channels.get(channel);
      if (notifier) {
        await notifier.send(status, alert);
      }
    }
  }

  getChannelsForAlert(alert) {
    const channels = ['email'];  // 默认邮件

    switch (alert.severity) {
      case 'critical':
        channels.push('dingtalk', 'sms', 'phone');
        break;
      case 'warning':
        channels.push('dingtalk');
        break;
    }

    return channels;
  }
}

// 告警聚合：避免告警风暴
class AlertAggregator {
  constructor() {
    this.groupWindow = 5 * 60 * 1000;  // 5分钟
    this.pendingAlerts = new Map();
  }

  groupAlert(alert) {
    // 按告警类型+标签分组
    const groupKey = this.getGroupKey(alert);

    if (!this.pendingAlerts.has(groupKey)) {
      this.pendingAlerts.set(groupKey, {
        count: 1,
        firstAlert: alert,
        lastAlert: new Date()
      });
      return ['send', alert];
    }

    const group = this.pendingAlerts.get(groupKey);
    group.count += 1;
    group.lastAlert = new Date();

    return ['group', group];
  }

  getGroupKey(alert) {
    return `${alert.name}:${JSON.stringify(alert.labels)}`;
  }
}

告警自愈机制

对于可自动修复的问题，实现告警自愈机制：

// 告警自愈执行器
class AlertSelfHealing {
  constructor() {
    this.actions = new Map();
    this.registerActions();
  }

  registerActions() {
    // 重启服务
    this.actions.set('restart_service', async (params) => {
      await systemControl.restart(params.serviceName);
    });

    // 清理磁盘空间
    this.actions.set('cleanup_disk', async (params) => {
      await diskCleaner.cleanOldLogs(params.days);
      await diskCleaner.cleanTempFiles();
    });

    // 重启数据库连接池
    this.actions.set('reset_db_pool', async () => {
      await dbPool.reset();
    });

    // 刷新缓存
    this.actions.set('flush_cache', async (params) => {
      await cacheManager.flush(params.pattern);
    });

    // 下线问题节点（K8s）
    this.actions.set('drain_node', async (params) => {
      await k8s.drainNode(params.nodeName);
      await k8s.cordonNode(params.nodeName);
    });
  }

  async execute(alert, actionConfig) {
    const action = this.actions.get(actionConfig.type);
    if (!action) {
      console.error(`Unknown action: ${actionConfig.type}`);
      return;
    }

    const execution = {
      alertId: alert.id,
      action: actionConfig.type,
      params: actionConfig.params,
      startTime: new Date(),
      status: 'running'
    };

    try {
      await action(actionConfig.params);
      execution.status = 'success';
      execution.endTime = new Date();

      // 记录成功执行的恢复动作
      await AlertHistory.recordRecovery(alert, execution);
    } catch (error) {
      execution.status = 'failed';
      execution.error = error.message;
      execution.endTime = new Date();

      // 发送告警通知给运维人员
      await AlertNotifier.notifySelfHealingFailed(alert, execution);
    }
  }
}

监控面板设计

监控面板展示系统运行状态：

• 系统概览 - CPU、内存、磁盘、网络等基础设施指标
• 应用监控 - QPS、响应时间、错误率等应用指标
• 数据库监控 - 连接数、慢查询、缓存命中率等
• 业务监控 - 活跃用户、表单提交量、流程处理量等
• 告警中心 - 当前告警、历史告警、告警统计分析

总结

完善的监控告警体系是保障 IMS 系统稳定运行的基础：

• 全面指标采集 - 覆盖基础设施、应用层、业务层
• 多维度告警 - 支持阈值、趋势、复合条件告警
• 告警聚合 - 避免告警风暴，减少运维噪音
• 多渠道通知 - 根据告警级别选择合适通知方式
• 告警自愈 - 自动处理常见问题，减少人工干预
• 可视化面板 - 直观展示系统状态，辅助运维决策