p99时延统计

性能数据

一般统计后端数据库的性能指标有最大时延，平均时延，但还有个更重要的指标即P99时延，99%的用户落在哪个时延区间

1
2
3
4
5
6
7
8
9
Latency    Number of requests
1s         5
2s         5
3s         10
4s         40
5s         20
6s         15
7s         4
8s         1

比如100个请求，第99个请求落在了7s这个区间,那么p99就是7秒,而不是前99个请求的平均时延

算法

  • 对时间划分区间，统计落在每个区间的请求数，和总时延
  • 每秒根据请求总数计算第99%的请求落在哪个时延区间
  • 计算时延区间的平均时延

比如，数据库的大部分时延在毫秒级，如此划分时延区间

  • 时延 [0-20ms)的1ms一个桶，总共20个桶
  • 时延 [20ms-200ms)的时候10ms一个桶，总共18个桶
  • 时延 [200ms-1000ms)50ms一个桶，总共16个桶
  • 时延 [1000ms+) 总共一个桶

某个请求的时延落在哪个桶类，则该桶中的请求总数++，时延总数++ （下章节的SetP99Delay函数）;每秒根据请求总数的99%,落在哪个时延区间,计算时延区间的平均时延（下章节的GetP99Delay函数）

实现

某监控统计实现，计算p99和获取p99关注函数

  • SetP99Delay
  • GetP99Delay
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
// .h头文件

#define BUCKET_20MS_SIZE 20    // [0-20ms)的1ms一个桶
#define BUCKET_200MS_SIZE 18   // [20ms-200ms)的时候10ms一个桶
#define BUCKET_1000MS_SIZE 16  // [200ms-1000ms)50ms一个桶
#define BUCKET_OTHER_SIZE 1    // [1000ms+) 一个桶

struct Bucket {
  int64_t totalDelay;
  int64_t count;
};

class Monitor {
 public:
  Monitor();
  ~Monitor();
  int Init(std::string cmd);

  void IncSendErrCount() {
    m_sendCount++;
    m_sendErrCount++;

    m_sendSumCount++;
    m_sendErrSumCount++;
  }
  void IncSendSuccCount() {
    m_sendCount++;

    m_sendSumCount++;
  }
  void IncRecvErrCount() {
    m_recvErrCount++;
    m_recvCount++;

    m_recvSumCount++;
    m_recvErrSumCount++;
  }

  void IncRecvSuccCount(int64_t delay) {
    m_recvCount++;
    m_recvSumCount++;
    if (delay < 0) {
      return;
    }

    m_totalDelay += delay;
    m_delayCount++;
    m_totalSumDelay += delay;
    m_delaySumCount++;
    if (m_maxDelay < delay) {
      m_maxDelay = delay;
    }
    if (m_maxSumDelay < delay) {
      m_maxSumDelay = delay;
    }
    SetP99Delay(delay);
  }
  void Print(int64_t curTimeMs);

 private:
  void Reset();
  void SetP99Delay(int64_t delay);
  void GetP99Delay(int64_t &p99Delay, int64_t &p999Delay, int64_t &p9999Delay);

 private:
  // period
  std::string m_cmd;
  int64_t m_sendCount;
  int64_t m_sendErrCount;
  int64_t m_recvCount;
  int64_t m_recvErrCount;
  int64_t m_delayCount;
  int64_t m_totalDelay;  // us
  int64_t m_maxDelay;    // us

  // sum
  int64_t m_sendSumCount;
  int64_t m_sendErrSumCount;
  int64_t m_recvSumCount;
  int64_t m_recvErrSumCount;
  int64_t m_delaySumCount;
  int64_t m_totalSumDelay;  // us
  int64_t m_maxSumDelay;    // us

  // P99计算，越小越精确
  // [0-20ms)的1ms一个桶，[20ms-200ms)的时候10ms一个桶
  // [200ms-1000ms)50ms一个桶，[1000ms+)单独一个桶
  Bucket m_20MSBucket[BUCKET_20MS_SIZE];
  int64_t m_20MSTotalRecvCount;
  Bucket m_200MSBucket[BUCKET_200MS_SIZE];
  int64_t m_200MSTotalRecvCount;
  Bucket m_1000MSBucket[BUCKET_1000MS_SIZE];
  int64_t m_1000MSTotalRecvCount;
  Bucket m_otherBucket;
};
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
// .cpp文件
Monitor::Monitor() {
  m_sendSumCount = 0;
  m_sendErrSumCount = 0;
  m_recvSumCount = 0;
  m_recvErrSumCount = 0;
  m_delaySumCount = 0;
  m_totalSumDelay = 0;
  m_maxSumDelay = 0;
  Reset();
}

Monitor::~Monitor() {}

int Monitor::Init(std::string cmd) {
  m_cmd = cmd;
  return 0;
}

void Monitor::Reset() {
  m_sendCount = 0;
  m_sendErrCount = 0;
  m_recvCount = 0;
  m_recvErrCount = 0;
  m_delayCount = 0;
  m_totalDelay = 0;
  m_maxDelay = 0;

  memset(m_20MSBucket, 0, sizeof(m_20MSBucket));
  m_20MSTotalRecvCount = 0;
  memset(m_200MSBucket, 0, sizeof(m_200MSBucket));
  m_200MSTotalRecvCount = 0;
  memset(m_1000MSBucket, 0, sizeof(m_1000MSBucket));
  m_1000MSTotalRecvCount = 0;
  memset(&m_otherBucket, 0, sizeof(m_otherBucket));
}

void Monitor::SetP99Delay(int64_t delay) {
  if (delay < 20000LL) {
    // [0-20ms)的1ms一个桶
    m_20MSBucket[delay / 1000LL].count++;
    m_20MSBucket[delay / 1000LL].totalDelay += delay;
    m_20MSTotalRecvCount++;
  } else if (delay < 200000LL) {
    // [20ms-200ms)的时候10ms一个桶
    m_200MSBucket[(delay - 20000LL) / 10000LL].count++;
    m_200MSBucket[(delay - 20000LL) / 10000LL].totalDelay += delay;
    m_200MSTotalRecvCount++;
  } else if (delay < 1000000LL) {
    // [200ms-1000ms)50ms一个桶
    m_1000MSBucket[(delay - 200000LL) / 50000LL].count++;
    m_1000MSBucket[(delay - 200000LL) / 50000LL].totalDelay += delay;
    m_1000MSTotalRecvCount++;
  } else {
    m_otherBucket.count++;
    m_otherBucket.totalDelay += delay;
  }
}

void Monitor::GetP99Delay(int64_t &p99Delay, int64_t &p999Delay, int64_t &p9999Delay) {
  p99Delay = 0;
  p999Delay = 0;
  p9999Delay = 0;
  if (m_delayCount <= 0) {
    return;
  }

  int64_t p99Count = m_delayCount * 99 / 100;
  int64_t p999Count = m_delayCount * 999 / 1000;
  int64_t p9999Count = m_delayCount * 9999 / 10000;
  DEBUG_LOG(BenchmarkLogger::GetInstance()->GetMonitorLogger(),
            "p99Count:{} p999Count:{}  p9999Count:{} m_20MSTotalRecvCount:{} m_200MSTotalRecvCount:{} "
            "m_1000MSTotalRecvCount:{}, m_otherBucket:{}",
            p99Count,
            p999Count,
            p9999Count,
            m_20MSTotalRecvCount,
            m_200MSTotalRecvCount,
            m_1000MSTotalRecvCount,
            m_otherBucket.count);
  if (m_20MSTotalRecvCount > 0) {
    // find [0-20ms)的1ms一个桶 m_20MSBucket
    int total = 0;
    int pos = 0;
    for (pos = 0; pos < BUCKET_20MS_SIZE; pos++) {
      if (m_20MSBucket[pos].count <= 0) {
        continue;
      }
      total += m_20MSBucket[pos].count;
      // p99
      if (total >= p99Count && p99Delay == 0) {
        p99Delay = m_20MSBucket[pos].totalDelay / m_20MSBucket[pos].count;
        if (m_20MSTotalRecvCount < p999Count) {
          break;
        }
      }

      // p999
      if (total >= p999Count && p999Delay == 0) {
        p999Delay = m_20MSBucket[pos].totalDelay / m_20MSBucket[pos].count;
        if (m_20MSTotalRecvCount < p9999Count) {
          break;
        }
      }

      // p9999
      if (total >= p9999Count && p9999Delay == 0) {
        p9999Delay = m_20MSBucket[pos].totalDelay / m_20MSBucket[pos].count;
        INFO_LOG(BenchmarkLogger::GetInstance()->GetMonitorLogger(),
                 "total:{} pos:{} totalDelay:{} count:{} p99Delay:{} p999Delay:{} p9999Delay:{}",
                 total,
                 pos,
                 m_20MSBucket[pos].totalDelay,
                 m_20MSBucket[pos].count,
                 p99Delay,
                 p999Delay,
                 p9999Delay);
        return;
      }
    }
  }

  if (m_200MSTotalRecvCount > 0) {
    // find [20ms-200ms)的时候10ms一个桶 m_200MSBucket
    int total = m_20MSTotalRecvCount;
    int pos = 0;
    for (pos = 0; pos < BUCKET_200MS_SIZE; pos++) {
      if (m_200MSBucket[pos].count <= 0) {
        continue;
      }
      total += m_200MSBucket[pos].count;

      // p99
      if (total >= p99Count && p99Delay == 0) {
        p99Delay = m_200MSBucket[pos].totalDelay / m_200MSBucket[pos].count;
        if (m_20MSTotalRecvCount + m_200MSTotalRecvCount < p999Count) {
          break;
        }
      }

      // p999
      if (total >= p999Count && p999Delay == 0) {
        p999Delay = m_200MSBucket[pos].totalDelay / m_200MSBucket[pos].count;
        if (m_20MSTotalRecvCount + m_200MSTotalRecvCount < p9999Count) {
          break;
        }
      }

      // p9999
      if (total >= p9999Count && p9999Delay == 0) {
        p9999Delay = m_200MSBucket[pos].totalDelay / m_200MSBucket[pos].count;
        INFO_LOG(BenchmarkLogger::GetInstance()->GetMonitorLogger(),
                 "total:{} pos:{} totalDelay:{} count:{} p99Delay:{} p999Delay:{} p9999Delay:{}",
                 total,
                 pos,
                 m_20MSBucket[pos].totalDelay,
                 m_20MSBucket[pos].count,
                 p99Delay,
                 p999Delay,
                 p9999Delay);
        return;
      }
    }
  }

  if (m_1000MSTotalRecvCount > 0) {
    // find [200ms-1000ms)50ms一个桶 m_1000MSBucket
    int total = m_20MSTotalRecvCount + m_200MSTotalRecvCount;
    int pos = 0;
    for (pos = 0; pos < BUCKET_1000MS_SIZE; pos++) {
      if (m_1000MSBucket[pos].count <= 0) {
        continue;
      }
      total += m_1000MSBucket[pos].count;

      // p99
      if (total >= p99Count && p99Delay == 0) {
        p99Delay = m_1000MSBucket[pos].totalDelay / m_1000MSBucket[pos].count;
        if (m_20MSTotalRecvCount + m_200MSTotalRecvCount + m_1000MSTotalRecvCount < p999Count) {
          break;
        }
      }

      // p999
      if (total >= p999Count && p999Delay == 0) {
        p999Delay = m_1000MSBucket[pos].totalDelay / m_1000MSBucket[pos].count;
        if (m_20MSTotalRecvCount + m_200MSTotalRecvCount + m_1000MSTotalRecvCount < p9999Count) {
          break;
        }
      }

      // p9999
      if (total >= p9999Count && p9999Delay == 0) {
        p9999Delay = m_1000MSBucket[pos].totalDelay / m_1000MSBucket[pos].count;
        INFO_LOG(BenchmarkLogger::GetInstance()->GetMonitorLogger(),
                 "total:{} pos:{} totalDelay:{} count:{} p99Delay:{} p999Delay:{} p9999Delay:{}",
                 total,
                 pos,
                 m_20MSBucket[pos].totalDelay,
                 m_20MSBucket[pos].count,
                 p99Delay,
                 p999Delay,
                 p9999Delay);
        return;
      }
    }
  }

  if (m_otherBucket.count > 0) {
    int64_t otherAvgDelay = m_otherBucket.totalDelay / m_otherBucket.count;
    if (0 == p99Delay) {
      p99Delay = otherAvgDelay;
    }
    if (0 == p999Delay) {
      p999Delay = otherAvgDelay;
    }
    if (0 == p9999Delay) {
      p9999Delay = otherAvgDelay;
    }
  }
  DEBUG_LOG(BenchmarkLogger::GetInstance()->GetMonitorLogger(), "p9999Delay:{}", p9999Delay);
}

void Monitor::Print(int64_t curTimeMs) {
  // 计算
  int64_t qps = m_sendCount - m_sendErrCount;
  int64_t avgDelay = 0;
  if (m_delayCount > 0) {
    avgDelay = m_totalDelay / m_delayCount;
  }
  int64_t p99Delay = 0;
  int64_t p999Delay = 0;
  int64_t p9999Delay = 0;
  GetP99Delay(p99Delay, p999Delay, p9999Delay);

  if (m_cmd.empty()) {
    // total
    time_t now = curTimeMs / 1000;
    tm *ltm = localtime(&now);
    printf(
        "{\"time\":\"%d-%.2d-%.2d %.2d:%.2d:%.2d\",\"qps\":%ld, "
        "\"request_count\":%ld,\"success_count\":%ld,\"failed_count\":%ld,\"average_delay\":\"%ld.%ldms\","
        "\"maximum_delay\":\"%ld.%ldms\",\"p99_delay\":\"%ld.%ldms\",\"p999_delay\":\"%ld.%ldms\",\"p9999_"
        "delay\":\"%ld.%ldms\",\"metrics_by_command\":[",
        1900 + ltm->tm_year,
        1 + ltm->tm_mon,
        ltm->tm_mday,
        ltm->tm_hour,
        ltm->tm_min,
        ltm->tm_sec,
        qps,
        qps,
        (m_recvCount - m_recvErrCount),
        (m_recvErrCount + m_sendErrCount),
        avgDelay / 1000,
        avgDelay % 1000,
        m_maxDelay / 1000,
        m_maxDelay % 1000,
        p99Delay / 1000,
        p99Delay % 1000,
        p999Delay / 1000,
        p999Delay % 1000,
        p9999Delay / 1000,
        p9999Delay % 1000);
  } else {
    INFO_LOG(BenchmarkLogger::GetInstance()->GetMonitorLogger(),
             "QPS:{} CMD:{} SendSucc:{} SendErr:{} Recv:{} RecvErr:{} AvgDelay:{}us MaxDelay:{}us "
             "P99Delay:{}us P999Delay:{}us P9999Delay:{}us",
             qps,
             m_cmd.c_str(),
             qps,
             m_sendErrCount,
             (m_recvCount - m_recvErrCount),
             m_recvErrCount,
             avgDelay,
             m_maxDelay,
             p99Delay,
             p999Delay,
             p9999Delay);
    printf(
        "{\"command\":\"%s\",\"qps\":%ld, "
        "\"request_count\":%ld,\"success_count\":%ld,\"failed_count\":%ld,\"average_delay\":\"%ld.%ldms\","
        "\"maximum_delay\":\"%ld.%ldms\",\"p99_delay\":\"%ld.%ldms\",\"p999_delay\":\"%ld.%ldms\",\"p9999_"
        "delay\":\"%ld.%ldms\"}",
        m_cmd.c_str(),
        qps,
        qps,
        (m_recvCount - m_recvErrCount),
        (m_recvErrCount + m_sendErrCount),
        avgDelay / 1000,
        avgDelay % 1000,
        m_maxDelay / 1000,
        m_maxDelay % 1000,
        p99Delay / 1000,
        p99Delay % 1000,
        p999Delay / 1000,
        p999Delay % 1000,
        p9999Delay / 1000,
        p9999Delay % 1000);
  }
  Reset();
}
C++代码库
#C++ #压测 #P99时延计算 #性能
p99时延统计
http://example.com/2024/04/03/p99时延统计/
作者
Liu XinWei
发布于
2024年4月3日
许可协议