xenvm: arm64: add Xen Enlighten and event channel support

This commit adds support of Xen Enlighten page and initial support for
Xen event channels. It is needed for future Xen PV drivers
implementation.

Now enlighten page is mapped to the prepared memory area on
PRE_KERNEL_1 stage. In case of success event channel logic gets
inited and can be used ASAP after Zephyr start. Current implementation
allows to use only pre-defined event channels (PV console/XenBus) and
works only in single CPU mode (without VCPUOP_register_vcpu_info).
Event channel allocation will be implemented in future versions.

Signed-off-by: Dmytro Firsov <dmytro_firsov@epam.com>

arch/arm64/core/xen/CMakeLists.txt

@@ -8,3 +8,4 @@ zephyr_compile_options($<$<COMPILE_LANGUAGE:ASM>:-D__ASSEMBLY__>)
 zephyr_compile_options(-D__XEN_INTERFACE_VERSION__=0x00040e00)
 
 zephyr_library_sources(hypercall.S)
+zephyr_library_sources(enlighten.c)

arch/arm64/core/xen/enlighten.c

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2021 EPAM Systems
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <arch/arm64/hypercall.h>
+#include <xen/events.h>
+#include <xen/generic.h>
+#include <xen/public/xen.h>
+#include <xen/public/memory.h>
+
+#include <device.h>
+#include <init.h>
+#include <kernel.h>
+#include <kernel/thread.h>
+#include <logging/log.h>
+
+LOG_MODULE_REGISTER(xen_enlighten);
+
+/*
+ * During Xen Enlighten initialization we need to have allocated memory page,
+ * where hypervisor shared_info will be mapped. k_aligned_alloc() is not
+ * available on PRE_KERNEL_1 stage, so we will use statically allocated buffer,
+ * which will be casted to 'struct shared_info'. It is needed to initialize Xen
+ * event channels as soon as possible after start.
+ */
+static uint8_t shared_info_buf[XEN_PAGE_SIZE] __aligned(XEN_PAGE_SIZE);
+
+/* Remains NULL until mapping will be finished by Xen */
+shared_info_t *HYPERVISOR_shared_info;
+
+static int xen_map_shared_info(const shared_info_t *shared_page)
+{
+	struct xen_add_to_physmap xatp;
+
+	xatp.domid = DOMID_SELF;
+	xatp.idx = 0;
+	xatp.space = XENMAPSPACE_shared_info;
+	xatp.gpfn = (((xen_pfn_t) shared_page) >> XEN_PAGE_SHIFT);
+
+	return HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
+}
+
+static int xen_enlighten_init(const struct device *dev)
+{
+	ARG_UNUSED(dev);
+	int ret = 0;
+	shared_info_t *info = (shared_info_t *) shared_info_buf;
+
+	ret = xen_map_shared_info(info);
+	if (ret) {
+		LOG_ERR("%s: failed to map for Xen shared page, ret = %d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	/* Set value for globally visible pointer */
+	HYPERVISOR_shared_info = info;
+
+	ret = xen_events_init();
+	if (ret) {
+		LOG_ERR("%s: failed init Xen event channels, ret = %d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+SYS_INIT(xen_enlighten_init, PRE_KERNEL_1, CONFIG_KERNEL_INIT_PRIORITY_DEVICE);

drivers/xen/events.c

@@ -9,17 +9,176 @@
 #include <xen/public/event_channel.h>
 #include <xen/events.h>
 
+#include <errno.h>
+#include <kernel.h>
+#include <logging/log.h>
+
+LOG_MODULE_REGISTER(xen_events);
+
+extern shared_info_t *HYPERVISOR_shared_info;
+
+static evtchn_handle_t event_channels[EVTCHN_2L_NR_CHANNELS];
+
+static void empty_callback(void *data) { }
+
 void notify_evtchn(evtchn_port_t port)
 {
 	struct evtchn_send send;
 
-	if (port >= EVTCHN_2L_NR_CHANNELS) {
-		printk("%s: trying to send notify for invalid evtchn #%u\n",
-				__func__, port);
-		return;
-	}
+	__ASSERT(port < EVTCHN_2L_NR_CHANNELS,
+		"%s: trying to send notify for invalid evtchn #%u\n",
+		__func__, port);
 
 	send.port = port;
 
 	HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
 }
+
+int bind_event_channel(evtchn_port_t port, evtchn_cb_t cb, void *data)
+{
+	__ASSERT(port < EVTCHN_2L_NR_CHANNELS,
+		"%s: trying to bind invalid evtchn #%u\n",
+		__func__, port);
+	__ASSERT(cb != NULL, "%s: NULL callback for evtchn #%u\n",
+		__func__, port);
+
+
+	if (event_channels[port].cb != empty_callback)
+		LOG_WRN("%s: re-bind callback for evtchn #%u\n",
+				__func__, port);
+
+	event_channels[port].priv = data;
+	event_channels[port].cb = cb;
+
+	return 0;
+}
+
+int unbind_event_channel(evtchn_port_t port)
+{
+	__ASSERT(port < EVTCHN_2L_NR_CHANNELS,
+		"%s: trying to unbind invalid evtchn #%u\n",
+		__func__, port);
+
+	event_channels[port].cb = empty_callback;
+	event_channels[port].priv = NULL;
+
+	return 0;
+}
+
+int mask_event_channel(evtchn_port_t port)
+{
+	shared_info_t *s = HYPERVISOR_shared_info;
+
+	__ASSERT(port < EVTCHN_2L_NR_CHANNELS,
+		"%s: trying to mask invalid evtchn #%u\n",
+		__func__, port);
+
+
+	sys_bitfield_set_bit((mem_addr_t) s->evtchn_mask, port);
+
+	return 0;
+}
+
+int unmask_event_channel(evtchn_port_t port)
+{
+	shared_info_t *s = HYPERVISOR_shared_info;
+
+	__ASSERT(port < EVTCHN_2L_NR_CHANNELS,
+		"%s: trying to unmask invalid evtchn #%u\n",
+		__func__, port);
+
+	sys_bitfield_clear_bit((mem_addr_t) s->evtchn_mask, port);
+
+	return 0;
+}
+
+static void clear_event_channel(evtchn_port_t port)
+{
+	shared_info_t *s = HYPERVISOR_shared_info;
+
+	sys_bitfield_clear_bit((mem_addr_t) s->evtchn_pending, port);
+}
+
+static inline xen_ulong_t get_pending_events(xen_ulong_t pos)
+{
+	shared_info_t *s = HYPERVISOR_shared_info;
+
+	return (s->evtchn_pending[pos] & ~(s->evtchn_mask[pos]));
+}
+
+static void process_event(evtchn_port_t port)
+{
+	evtchn_handle_t channel = event_channels[port];
+
+	clear_event_channel(port);
+	channel.cb(channel.priv);
+}
+
+static void events_isr(void *data)
+{
+	ARG_UNUSED(data);
+
+	/* Needed for 2-level unwrapping */
+	xen_ulong_t pos_selector;   /* bits are positions in pending array */
+	xen_ulong_t events_pending; /* bits - events in pos_selector element */
+	uint32_t pos_index, event_index; /* bit indexes */
+
+	evtchn_port_t port; /* absolute event index */
+
+	/* TODO: SMP? XEN_LEGACY_MAX_VCPUS == 1*/
+	vcpu_info_t *vcpu = &HYPERVISOR_shared_info->vcpu_info[0];
+
+	/*
+	 * Need to set it to 0 /before/ checking for pending work, thus
+	 * avoiding a set-and-check race (check struct vcpu_info_t)
+	 */
+	vcpu->evtchn_upcall_pending = 0;
+
+	compiler_barrier();
+
+	/* Can not use system atomic_t/atomic_set() due to 32-bit casting */
+	pos_selector = __atomic_exchange_n(&vcpu->evtchn_pending_sel,
+					0, __ATOMIC_SEQ_CST);
+
+	while (pos_selector) {
+		/* Find first position, clear it in selector and process */
+		pos_index = __builtin_ffsl(pos_selector) - 1;
+		pos_selector &= ~(1 << pos_index);
+
+		/* Find all active evtchn on selected position */
+		while ((events_pending = get_pending_events(pos_index)) != 0) {
+			event_index =  __builtin_ffsl(events_pending) - 1;
+			events_pending &= (1 << event_index);
+
+			port = (pos_index * 8 * sizeof(xen_ulong_t))
+					+ event_index;
+			process_event(port);
+		}
+	}
+}
+
+int xen_events_init(void)
+{
+	int i;
+
+	if (!HYPERVISOR_shared_info) {
+		/* shared info was not mapped */
+		LOG_ERR("%s: shared_info - NULL, can't setup events\n", __func__);
+		return -EINVAL;
+	}
+
+	/* bind all ports with default callback */
+	for (i = 0; i < EVTCHN_2L_NR_CHANNELS; i++) {
+		event_channels[i].cb = empty_callback;
+		event_channels[i].priv = NULL;
+	}
+
+	IRQ_CONNECT(DT_IRQ_BY_IDX(DT_INST(0, xen_xen), 0, irq),
+		DT_IRQ_BY_IDX(DT_INST(0, xen_xen), 0, priority), events_isr,
+		NULL, DT_IRQ_BY_IDX(DT_INST(0, xen_xen), 0, flags));
+
+	irq_enable(DT_IRQ_BY_IDX(DT_INST(0, xen_xen), 0, irq));
+
+	LOG_INF("%s: events inited\n", __func__);
+	return 0;
+}

include/xen/events.h

@@ -8,6 +8,21 @@
 
 #include <xen/public/event_channel.h>
 
+#include <kernel.h>
+
+typedef void (*evtchn_cb_t)(void *priv);
+
+struct event_channel_handle {
+	evtchn_cb_t cb;
+	void *priv;
+};
+
+typedef struct event_channel_handle evtchn_handle_t;
+
 void notify_evtchn(evtchn_port_t port);
+int bind_event_channel(evtchn_port_t port, evtchn_cb_t cb, void *data);
+int unbind_event_channel(evtchn_port_t port);
+
+int xen_events_init(void);
 
 #endif /* __XEN_EVENTS_H__ */