cfn/computing-offload
Code Issues Proposed changes
computing-offload/generic_vdpa/qemu/0007-vhost-implement-savevm_handler-for-vdpa-device.patch
jiangdongxu fac818adb1 vdpa: change libvirt/qemu/kernel base version and docs 
Change-Id: I574115f5e44e56ac2baaad71eeab6b9fd7149747
2024-10-15 17:39:52 +08:00

270 lines
7.9 KiB
Diff
Raw Permalink Blame History

From 8da98e39ec193012f1c3a078e08502f376f28fac Mon Sep 17 00:00:00 2001
From: libai <libai12@huawei.com>
Date: Mon, 4 Dec 2023 15:53:28 +0800
Subject: [PATCH 07/16] vhost: implement savevm_handler for vdpa device
Register savevm_handler ops for vdpa devices to support migration:x
Signed-off-by: libai <libai12@huawei.com>
---
hw/virtio/vdpa-dev-mig.c | 175 +++++++++++++++++++++++++++++++
include/hw/virtio/vdpa-dev-mig.h | 13 +++
linux-headers/linux/vhost.h | 8 ++
3 files changed, 196 insertions(+)
diff --git a/hw/virtio/vdpa-dev-mig.c b/hw/virtio/vdpa-dev-mig.c
index 1d2bed2..662d4a2 100644
--- a/hw/virtio/vdpa-dev-mig.c
+++ b/hw/virtio/vdpa-dev-mig.c
@@ -21,9 +21,21 @@
#include "hw/virtio/vhost.h"
#include "hw/virtio/vdpa-dev.h"
#include "hw/virtio/virtio-bus.h"
+#include "migration/register.h"
#include "migration/migration.h"
#include "qemu/error-report.h"
#include "hw/virtio/vdpa-dev-mig.h"
+#include "migration/qemu-file-types.h"
+
+/*
+ * Flags used as delimiter:
+ * 0xffffffff => MSB 32-bit all 1s
+ * 0xef10 => emulated (virtual) function IO
+ * 0x0000 => 16-bits reserved for flags
+ */
+#define VDPA_MIG_FLAG_END_OF_STATE (0xffffffffef100001ULL)
+#define VDPA_MIG_FLAG_DEV_CONFIG_STATE (0xffffffffef100002ULL)
+#define VDPA_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL)
static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request,
void *arg)
@@ -39,6 +51,80 @@ static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request,
return ioctl(fd, request, arg);
}
+static int vhost_vdpa_set_mig_state(struct vhost_dev *dev, uint8_t state)
+{
+ return vhost_vdpa_call(dev, VHOST_VDPA_SET_MIG_STATE, &state);
+}
+
+static int vhost_vdpa_dev_buffer_size(struct vhost_dev *dev, uint32_t *size)
+{
+ return vhost_vdpa_call(dev, VHOST_GET_DEV_BUFFER_SIZE, size);
+}
+
+static int vhost_vdpa_dev_buffer_save(struct vhost_dev *dev, QEMUFile *f)
+{
+ struct vhost_vdpa_config *config;
+ unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
+ uint32_t buffer_size = 0;
+ int ret;
+
+ ret = vhost_vdpa_dev_buffer_size(dev, &buffer_size);
+ if (ret) {
+ error_report("get dev buffer size failed: %d\n", ret);
+ return ret;
+ }
+
+ qemu_put_be32(f, buffer_size);
+
+ config = g_malloc(buffer_size + config_size);
+ config->off = 0;
+ config->len = buffer_size;
+
+ ret = vhost_vdpa_call(dev, VHOST_GET_DEV_BUFFER, config);
+ if (ret) {
+ error_report("get dev buffer failed: %d\n", ret);
+ goto free;
+ }
+
+ qemu_put_buffer(f, config->buf, buffer_size);
+free:
+ g_free(config);
+
+ return ret;
+}
+
+static int vhost_vdpa_dev_buffer_load(struct vhost_dev *dev, QEMUFile *f)
+{
+ struct vhost_vdpa_config *config;
+ unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
+ uint32_t buffer_size, recv_size;
+ int ret;
+
+ buffer_size = qemu_get_be32(f);
+
+ config = g_malloc(buffer_size + config_size);
+ config->off = 0;
+ config->len = buffer_size;
+
+ recv_size = qemu_get_buffer(f, config->buf, buffer_size);
+ if (recv_size != buffer_size) {
+ error_report("read dev mig buffer failed, buffer_size: %u, "
+ "recv_size: %u\n", buffer_size, recv_size);
+ ret = -EINVAL;
+ goto free;
+ }
+
+ ret = vhost_vdpa_call(dev, VHOST_SET_DEV_BUFFER, config);
+ if (ret) {
+ error_report("set dev buffer failed: %d\n", ret);
+ }
+
+free:
+ g_free(config);
+
+ return ret;
+}
+
static int vhost_vdpa_device_suspend(VhostVdpaDevice *vdpa)
{
VirtIODevice *vdev = VIRTIO_DEVICE(vdpa);
@@ -165,14 +251,103 @@ static void vdpa_dev_vmstate_change(void *opaque, bool running, RunState state)
}
}
+static int vdpa_save_setup(QEMUFile *f, void *opaque)
+{
+ qemu_put_be64(f, VDPA_MIG_FLAG_DEV_SETUP_STATE);
+ qemu_put_be64(f, VDPA_MIG_FLAG_END_OF_STATE);
+
+ return qemu_file_get_error(f);
+}
+
+static int vdpa_save_complete_precopy(QEMUFile *f, void *opaque)
+{
+ VhostVdpaDevice *vdev = VHOST_VDPA_DEVICE(opaque);
+ struct vhost_dev *hdev = &vdev->dev;
+ int ret;
+
+ qemu_put_be64(f, VDPA_MIG_FLAG_DEV_CONFIG_STATE);
+ ret = vhost_vdpa_dev_buffer_save(hdev, f);
+ if (ret) {
+ error_report("Save vdpa device buffer failed: %d\n", ret);
+ return ret;
+ }
+ qemu_put_be64(f, VDPA_MIG_FLAG_END_OF_STATE);
+
+ return qemu_file_get_error(f);
+}
+
+static int vdpa_load_state(QEMUFile *f, void *opaque, int version_id)
+{
+ VhostVdpaDevice *vdev = VHOST_VDPA_DEVICE(opaque);
+ struct vhost_dev *hdev = &vdev->dev;
+
+ int ret;
+ uint64_t data;
+
+ data = qemu_get_be64(f);
+ while (data != VDPA_MIG_FLAG_END_OF_STATE) {
+ if (data == VDPA_MIG_FLAG_DEV_SETUP_STATE) {
+ data = qemu_get_be64(f);
+ if (data == VDPA_MIG_FLAG_END_OF_STATE) {
+ return 0;
+ } else {
+ error_report("SETUP STATE: EOS not found 0x%lx\n", data);
+ return -EINVAL;
+ }
+ } else if (data == VDPA_MIG_FLAG_DEV_CONFIG_STATE) {
+ ret = vhost_vdpa_dev_buffer_load(hdev, f);
+ if (ret) {
+ error_report("fail to restore device buffer.\n");
+ return ret;
+ }
+ }
+
+ ret = qemu_file_get_error(f);
+ if (ret) {
+ error_report("qemu file error: %d\n", ret);
+ return ret;
+ }
+ data = qemu_get_be64(f);
+ }
+
+ return 0;
+}
+
+static int vdpa_load_setup(QEMUFile *f, void *opaque)
+{
+ VhostVdpaDevice *v = VHOST_VDPA_DEVICE(opaque);
+ struct vhost_dev *hdev = &v->dev;
+ int ret = 0;
+
+ ret = vhost_vdpa_set_mig_state(hdev, VDPA_DEVICE_PRE_START);
+ if (ret) {
+ error_report("pre start device failed: %d\n", ret);
+ goto out;
+ }
+
+ return qemu_file_get_error(f);
+out:
+ return ret;
+}
+
+static SaveVMHandlers savevm_vdpa_handlers = {
+ .save_setup = vdpa_save_setup,
+ .save_live_complete_precopy = vdpa_save_complete_precopy,
+ .load_state = vdpa_load_state,
+ .load_setup = vdpa_load_setup,
+};
+
void vdpa_migration_register(VhostVdpaDevice *vdev)
{
vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev),
vdpa_dev_vmstate_change,
DEVICE(vdev));
+ register_savevm_live("vdpa", -1, 1,
+ &savevm_vdpa_handlers, DEVICE(vdev));
}
void vdpa_migration_unregister(VhostVdpaDevice *vdev)
{
+ unregister_savevm(VMSTATE_IF(&vdev->parent_obj.parent_obj), "vdpa", DEVICE(vdev));
qemu_del_vm_change_state_handler(vdev->vmstate);
}
diff --git a/include/hw/virtio/vdpa-dev-mig.h b/include/hw/virtio/vdpa-dev-mig.h
index 89665ca..adc1d65 100644
--- a/include/hw/virtio/vdpa-dev-mig.h
+++ b/include/hw/virtio/vdpa-dev-mig.h
@@ -9,6 +9,19 @@
#include "hw/virtio/vdpa-dev.h"
+enum {
+ VDPA_DEVICE_START,
+ VDPA_DEVICE_STOP,
+ VDPA_DEVICE_PRE_START,
+ VDPA_DEVICE_PRE_STOP,
+ VDPA_DEVICE_CANCEL,
+ VDPA_DEVICE_POST_START,
+ VDPA_DEVICE_START_ASYNC,
+ VDPA_DEVICE_STOP_ASYNC,
+ VDPA_DEVICE_PRE_START_ASYNC,
+ VDPA_DEVICE_QUERY_OP_STATE,
+};
+
void vdpa_migration_register(VhostVdpaDevice *vdev);
void vdpa_migration_unregister(VhostVdpaDevice *vdev);
diff --git a/linux-headers/linux/vhost.h b/linux-headers/linux/vhost.h
index be145b4..814fd9c 100644
--- a/linux-headers/linux/vhost.h
+++ b/linux-headers/linux/vhost.h
@@ -223,4 +223,12 @@
*/
#define VHOST_VDPA_RESUME _IO(VHOST_VIRTIO, 0x7E)
+/* set and get device buffer */
+#define VHOST_GET_DEV_BUFFER _IOR(VHOST_VIRTIO, 0xb0, struct vhost_vdpa_config)
+#define VHOST_SET_DEV_BUFFER _IOW(VHOST_VIRTIO, 0xb1, struct vhost_vdpa_config)
+#define VHOST_GET_DEV_BUFFER_SIZE _IOR(VHOST_VIRTIO, 0xb3, __u32)
+
+/* set device migtration state */
+#define VHOST_VDPA_SET_MIG_STATE _IOW(VHOST_VIRTIO, 0xb2, __u8)
+
#endif
--
2.46.0.windows.1
View Git Blame Copy Permalink