一、falcon磁盘IO告警计算方法

(1)线上告警示例【falcon】环境: prod 时间: 2018-11-10 22:29 共1条【#主机磁盘io过高(appid)】主机hostname磁盘dfa io过高98.76>98%(2)cat /proc/diskstats252 0 dfa 2689642164 0 513826977162 100403006 6803204529 0 1348198360088 2893131263 0 934780608 3002409596每项的具体含义如下:1   252            主设备号    code:Major2   0              次设备号    code:Minor3   dfa            设备名称    code:Name 4   2689642164     <1>读完成次数{读磁盘的次数,成功完成读的总次数}[2689642164]  code:ReadIOs5   0              <2>合并读完成次数  code:ReadMerges6   513826977162   <3>读扇区的次数,成功读过的扇区总次数  5.ReadSectors7   100403006      <4>读花费的毫秒数;这是所有读操作所花费的毫秒数{从__make_request()到end_that_request_last()的测量} code:ReadTicks8   6803204529     <5>写完成次数;写完成的次数,成功写完成的总次数  code:WriteIOs9   0              <6>合并写完成次数。为了效率可能会合并相邻的读和写,从而两次4K的读在它最终被处理到磁盘上之前可能会变成一次8K的读,才被计数(和排队),因此只有一次I/O操作,这个域使你知道这样的操作有多频繁 code:WriteMerges10  1348198360088  <7>写扇区次数;写扇区的次数,成功写扇区总次数  code:WriteSectors11  2893131263     <8>写操作花费的毫秒数;写花费的毫秒数,这是所有写操作所花费的毫秒数{是从__make_request()到end_that_request_last()的测量}  code:WriteTicks12  0              <9>正在处理的输入/输出请求数;I/O的当前进度,只有这个域应该是0.当请求被交给适当的request_queue_t时增加和请求完成时减小 code:InFlight13  934780608      <10>输入/输出操作花费的毫秒数;花在I/O操作上的毫秒数,只要field9不为0这个域就会增长 code:IOTicks14  3002409596     <11>输入/输出操作花费的加权毫秒数;花在I/O操作上的毫秒数,在每次I/O开始,I/O结束,I/O合并或读取自动上次更新这个域以来(第<9>列正在进行的io数量乘以花费在io上的毫秒数)时这个域都会增加.这可以给I/O完成时间和存储那些可以累积的提供一个便利的测量标准 code:TimeInQueue(3)告警计算公式 io_ticks{crt.IOTicks - last.IOTicks(上10s的值)}/100 > 98 会进行报警

二、实现代码

(1)const diskStatPath = "/proc/diskstats"type DiskStatCollector struct {	lastDiskStats map[string]*linux.DiskStat}func (c *DiskStatCollector) Collect() ([]*model.Metric, error) {	if c.lastDiskStats == nil {		c.lastDiskStats = make(map[string]*linux.DiskStat)	}	disks, err := linux.ReadDiskStats(diskStatPath)	if err != nil {		return nil, fmt.Errorf("collect disk stat: %v", err)	}	var metrics []*model.Metric	for _, crt := range disks {		if len(crt.Name) == 3 && (strings.HasPrefix(crt.Name, "sd") || strings.HasPrefix(crt.Name, "vd") || strings.HasPrefix(crt.Name, "df")) {			if last, ok := c.lastDiskStats[crt.Name]; ok {				metrics = append(metrics, diskStatDiff(&crt, last)...)			}			tmp := crt			c.lastDiskStats[crt.Name] = &tmp		}	}	return metrics, nil}func diskStatDiff(crt *linux.DiskStat, last *linux.DiskStat) []*model.Metric {	var metrics []*model.Metric	if crt.IOTicks < last.IOTicks {		return metrics	}	metrics = append(metrics, &model.Metric{		Name: "disk.io",		Fields: []*model.Field{			{"read_ios", model.Gauge, crt.ReadIOs - last.ReadIOs},			{"read_merges", model.Gauge, crt.ReadMerges - last.ReadMerges},			{"read_sectors", model.Gauge, crt.ReadSectors - last.ReadSectors},			{"read_ticks", model.Gauge, crt.ReadTicks - last.ReadTicks},			{"write_ios", model.Gauge, crt.WriteIOs - last.WriteIOs},			{"write_merges", model.Gauge, crt.WriteMerges - last.WriteMerges},			{"write_sectors", model.Gauge, crt.WriteSectors - last.WriteSectors},			{"write_ticks", model.Gauge, crt.WriteTicks - last.WriteTicks},			{"in_flight", model.Gauge, crt.InFlight - last.InFlight},			{"io_ticks", model.Gauge, crt.IOTicks - last.IOTicks},			{"time_in_queue", model.Gauge, crt.TimeInQueue - last.TimeInQueue},		},		Tags:      map[string]string{"name": crt.Name},		Timestamp: utils.Timestamp(),	})	return metrics}(2)// ReadDiskStats reads and parses the file.//// Note:// * Assumes a well formed file and will panic if it isn't.func ReadDiskStats(path string) ([]DiskStat, error) {	data, err := ioutil.ReadFile(path)	if err != nil {		return nil, err	}	devices := strings.Split(string(data), "\n")	results := make([]DiskStat, len(devices)-1)	for i := range results {		fields := strings.Fields(devices[i])		Major, _ := strconv.ParseInt(fields[0], 10, strconv.IntSize)		results[i].Major = int(Major)		Minor, _ := strconv.ParseInt(fields[1], 10, strconv.IntSize)		results[i].Minor = int(Minor)		results[i].Name = fields[2]		results[i].ReadIOs, _ = strconv.ParseUint(fields[3], 10, 64)		results[i].ReadMerges, _ = strconv.ParseUint(fields[4], 10, 64)		results[i].ReadSectors, _ = strconv.ParseUint(fields[5], 10, 64)		results[i].ReadTicks, _ = strconv.ParseUint(fields[6], 10, 64)		results[i].WriteIOs, _ = strconv.ParseUint(fields[7], 10, 64)		results[i].WriteMerges, _ = strconv.ParseUint(fields[8], 10, 64)		results[i].WriteSectors, _ = strconv.ParseUint(fields[9], 10, 64)		results[i].WriteTicks, _ = strconv.ParseUint(fields[10], 10, 64)		results[i].InFlight, _ = strconv.ParseUint(fields[11], 10, 64)		results[i].IOTicks, _ = strconv.ParseUint(fields[12], 10, 64)		results[i].TimeInQueue, _ = strconv.ParseUint(fields[13], 10, 64)	}	return results, nil}