libvirt-performance

Posted on Edited on In ,

Introduction

For performance issue, like a rpc all takes long time, you want to know which function takes much time without modifying source code and restart libvirt, also you want to collect stats of libvirt, here I will share the way to meet this reqirement, the core tool is systemtap, you can also refer to linux systemtap guide which shows how to enable systemtap and write probes. with systemtap, you can insert any code outside application without restarting it or rebuilding it

systemtap

Systemtap is a scripting language and tool for dynamically probing or tracing in Linux kernel space or user space. libvirt already has built in markers(static probes) and tapset to simple your use, this feature can be enabled when compile libvirt

1
2
$ ../configure --with-dtrace
$ make

The tapset is installed at /usr/share/systemtap/tapset/libvirt_* if enabled.

NOTE:

With dtrace enabled

when dtrace enabled, you can use the markers registered in source code and probe on marker like this, debuginfo is not a must in this way, this is can used for production env!!!

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
# list all markers
$ grep 'probe libvirt.[a-z_0-9.]*' /usr/share/systemtap/tapset/libvirt_* -o
probe libvirt.event_poll.add_handle
probe libvirt.event_poll.update_handle
probe libvirt.event_poll.remove_handle
probe libvirt.event_poll.dispatch_handle
probe libvirt.event_poll.purge_handle
probe libvirt.event_poll.add_timeout
probe libvirt.event_poll.update_timeout
probe libvirt.event_poll.remove_timeout
probe libvirt.event_poll.dispatch_timeout
probe libvirt.event_poll.purge_timeout
probe libvirt.event_poll.run
probe libvirt.dbus.method_call
probe libvirt.dbus.method_error
probe libvirt.dbus.method_reply
probe libvirt.object.new
probe libvirt.object.ref
probe libvirt.object.unref
probe libvirt.object.dispose
probe libvirt.rpc.socket_new
probe libvirt.rpc.socket_dispose
probe libvirt.rpc.socket_send_fd
probe libvirt.rpc.socket_recv_fd
probe libvirt.rpc.server_client_new
probe libvirt.rpc.server_client_dispose
probe libvirt.rpc.server_client_msg_tx_queue
probe libvirt.rpc.server_client_msg_rx
probe libvirt.rpc.client_new
probe libvirt.rpc.client_dispose
probe libvirt.rpc.client_msg_tx_queue
probe libvirt.rpc.client_msg_rx
probe libvirt.rpc.server_client_auth_allow
probe libvirt.rpc.server_client_auth_deny
probe libvirt.rpc.server_client_auth_fail
probe libvirt.rpc.tls_context_new
probe libvirt.rpc.tls_context_dispose
probe libvirt.rpc.tls_context_session_allow
probe libvirt.rpc.tls_context_session_deny
probe libvirt.rpc.tls_context_session_fail
probe libvirt.rpc.tls_session_new
probe libvirt.rpc.tls_session_dispose
probe libvirt.rpc.tls_session_handshake_pass
probe libvirt.rpc.tls_session_handshake_fail
probe libvirt.rpc.keepalive_new
probe libvirt.rpc.keepalive_dispose
probe libvirt.rpc.keepalive_start
probe libvirt.rpc.keepalive_stop
probe libvirt.rpc.keepalive_send
probe libvirt.rpc.keepalive_received
probe libvirt.rpc.keepalive_timeout
probe libvirt.qemu.monitor_new
probe libvirt.qemu.monitor_ref
probe libvirt.qemu.monitor_unref
probe libvirt.qemu.monitor_close
probe libvirt.qemu.monitor_send_msg
probe libvirt.qemu.monitor_recv_reply
probe libvirt.qemu.monitor_recv_event
probe libvirt.qemu.monitor_io_process
probe libvirt.qemu.monitor_io_read
probe libvirt.qemu.monitor_io_write
probe libvirt.qemu.monitor_io_send_fd

# check the args for each marker in order to use that marker
# vi /usr/share/systemtap/tapset/libvirt_qemu_probes.stp
# vi /usr/share/systemtap/tapset/libvirt_probes.stp

Write probe on marker

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# stp is a script but similar like C
$ cat qmp.stp
probe begin
{
  printf("Start tracing\n");
}
# check marker parameters from above
probe libvirt.qemu.monitor_send_msg
{
  printf("%s QMPs: %s", tz_ctime(gettimeofday_s()), msg);
}

$ stap qmp.stp -x `pidof libvirtd`
Start tracing
Wed Jan  5 09:27:28 2022 CST QMPs: {"execute":"query-version","id":"libvirt-13"}

# OR enable probe when starting libvirtd
$ stap qmp.stp -c "/usr/sbin/libvirtd"

without dtrace enable

When dtrace is disabled, there is no markder available, you have to write probe by yourself and build with deubg mode or install debuginfo of libvirt and shared library used by libvirt, without dtrace from libvirt itself, you have more freedom and more control, as you can trace on any function, but more complex as well.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# must build libvirt and shared library with debuginfo first

$ cat qmp.stp
probe begin
{
  printf("Start tracing\n")
}
# check the function parameters
probe process("/usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so").function("qemuMonitorSend") {
  // can access fields like we did in C
  printf("%s, mon fd: %d, vm: %s, QMPs: %s", tz_ctime(gettimeofday_s()), $mon->fd, user_string($mon->vm->def->name), user_string($msg->txBuffer))
}

$ stap qmp.stp -x `pidof libvirtd`
Start tracing
Wed Jan  5 10:38:44 2022 CST, mon fd: 22, vm: centos, QMPs: {"execute":"query-version","id":"libvirt-32"}
# OR
$ stap qmp.stp -c "/usr/sbin/libvirtd"

perf

1
2
3
# perf on given process, include all fucntions related(kernel and userspace)
$ perf record -p 21692
$ perf report