diff --git a/input_manager.c b/input_manager.c
index 0eeb253..1385f25 100644
--- a/input_manager.c
+++ b/input_manager.c
@@ -92,12 +92,7 @@ input_manager_handle_device_destroy(struct wl_listener *listener, void *data) {
 }
 
 static void
-handle_new_input(struct wl_listener *listener, void *data) {
-	struct cg_input_manager *input =
-	    wl_container_of(listener, input, new_input);
-	struct cg_server *server = input->server;
-	struct wlr_input_device *device = data;
-
+new_input(struct cg_input_manager *input, struct wlr_input_device *device, bool virtual) {
 	struct cg_input_device *input_device =
 	    calloc(1, sizeof(struct cg_input_device));
 	if(!input_device) {
@@ -106,9 +101,10 @@ handle_new_input(struct wl_listener *listener, void *data) {
 	}
 	device->data = input_device;
 
+	input_device->is_virtual=virtual;
 	input_device->wlr_device = device;
 	input_device->identifier = input_device_get_identifier(device);
-	input_device->server = server;
+	input_device->server = input->server;
 	input_device->pointer = NULL;
 	input_device->touch = NULL;
 
@@ -119,39 +115,28 @@ handle_new_input(struct wl_listener *listener, void *data) {
 	wl_signal_add(&device->events.destroy, &input_device->device_destroy);
 	input_device->device_destroy.notify = input_manager_handle_device_destroy;
 
-	struct cg_seat *seat = server->seat;
+	struct cg_seat *seat = input->server->seat;
 	seat_add_device(seat, input_device);
 }
 
+static void
+handle_new_input(struct wl_listener *listener, void *data) {
+	struct cg_input_manager *input =
+	    wl_container_of(listener, input, new_input);
+	struct wlr_input_device *device = data;
+
+	new_input(input,device,false);
+}
+
 void
 handle_virtual_keyboard(struct wl_listener *listener, void *data) {
-	struct cg_input_manager *input_manager =
-	    wl_container_of(listener, input_manager, virtual_keyboard_new);
+
+	struct cg_input_manager *input =
+	    wl_container_of(listener, input, virtual_keyboard_new);
 	struct wlr_virtual_keyboard_v1 *keyboard = data;
 	struct wlr_input_device *device = &keyboard->input_device;
 
-	struct cg_seat *seat = input_manager->server->seat;
-
-	struct cg_input_device *input_device =
-	    calloc(1, sizeof(struct cg_input_device));
-	if(!input_device) {
-		wlr_log(WLR_ERROR, "could not allocate input device");
-		return;
-	}
-	device->data = input_device;
-
-	input_device->is_virtual = true;
-	input_device->wlr_device = device;
-	input_device->identifier = input_device_get_identifier(device);
-	wl_list_insert(&input_manager->devices, &input_device->link);
-	input_device->server = input_manager->server;
-	input_device->pointer = NULL;
-	input_device->touch = NULL;
-
-	wl_signal_add(&device->events.destroy, &input_device->device_destroy);
-	input_device->device_destroy.notify = input_manager_handle_device_destroy;
-
-	seat_add_device(seat, input_device);
+	new_input(input,device,true);
 }
 
 void
@@ -162,31 +147,10 @@ handle_virtual_pointer(struct wl_listener *listener, void *data) {
 	struct wlr_virtual_pointer_v1 *pointer = event->new_pointer;
 	struct wlr_input_device *device = &pointer->input_device;
 
-	struct cg_seat *seat = input_manager->server->seat;
-
-	struct cg_input_device *input_device =
-	    calloc(1, sizeof(struct cg_input_device));
-	if(!input_device) {
-		wlr_log(WLR_ERROR, "could not allocate input device");
-		return;
-	}
-	device->data = input_device;
-
-	input_device->is_virtual = true;
-	input_device->wlr_device = device;
-	input_device->identifier = input_device_get_identifier(device);
-	wl_list_insert(&input_manager->devices, &input_device->link);
-	input_device->server = input_manager->server;
-	input_device->pointer = NULL;
-	input_device->touch = NULL;
-
-	wl_signal_add(&device->events.destroy, &input_device->device_destroy);
-	input_device->device_destroy.notify = input_manager_handle_device_destroy;
-
-	seat_add_device(seat, input_device);
+	new_input(input_manager,device,true);
 
 	if(event->suggested_output) {
-		wlr_cursor_map_input_to_output(seat->cursor, device,
+		wlr_cursor_map_input_to_output(input_manager->server->seat->cursor, device,
 		                               event->suggested_output);
 	}
 }
diff --git a/seat.c b/seat.c
index 89b1eb8..8fd485e 100644
--- a/seat.c
+++ b/seat.c
@@ -326,15 +326,24 @@ handle_keyboard_group_modifiers(struct wl_listener *listener, void *_data) {
 	handle_modifier_event(group->wlr_group->input_device, group->seat);
 }
 
+static bool repeat_info_match(struct wlr_keyboard *a, struct wlr_keyboard *b) {
+	return a->repeat_info.rate == b->repeat_info.rate &&
+		a->repeat_info.delay == b->repeat_info.delay;
+}
+
 static void
-cg_keyboard_group_add(struct wlr_input_device *device, struct cg_seat *seat) {
+cg_keyboard_group_add(struct cg_input_device *input_device, struct cg_seat *seat) {
+	struct wlr_input_device *device=input_device->wlr_device;
 	struct wlr_keyboard *wlr_keyboard = device->keyboard;
-	struct cg_keyboard_group *group;
-	wl_list_for_each(group, &seat->keyboard_groups, link) {
-		struct wlr_keyboard_group *wlr_group = group->wlr_group;
-		if(wlr_keyboard_group_add_keyboard(wlr_group, wlr_keyboard)) {
-			wlr_log(WLR_DEBUG, "Adding keyboard to existing group.");
-			return;
+	// Virtual devices should not be grouped
+	if(!input_device->is_virtual) {
+		struct cg_keyboard_group *group;
+		wl_list_for_each(group, &seat->keyboard_groups, link) {
+			struct wlr_keyboard_group *wlr_group = group->wlr_group;
+			if(wlr_keyboard_keymaps_match(wlr_keyboard->keymap,wlr_group->keyboard.keymap) && repeat_info_match(wlr_keyboard,&wlr_group->keyboard) && wlr_keyboard_group_add_keyboard(wlr_group, wlr_keyboard)) {
+				wlr_log(WLR_DEBUG, "Adding keyboard to existing group.");
+				return;
+			}
 		}
 	}
 	/* This is reached if and only if the keyboard could not be inserted into
@@ -406,7 +415,7 @@ new_keyboard(struct cg_seat *seat, struct cg_input_device *input_device) {
 	xkb_context_unref(context);
 	wlr_keyboard_set_repeat_info(device->keyboard, 25, 600);
 
-	cg_keyboard_group_add(device, seat);
+	cg_keyboard_group_add(input_device, seat);
 	++seat->num_keyboards;
 
 	wlr_seat_set_keyboard(seat->seat, device);
@@ -436,6 +445,10 @@ seat_add_device(struct cg_seat *seat, struct cg_input_device *device) {
 	update_capabilities(seat);
 }
 
+static void destroy_empty_wlr_keyboard_group(void *data) {
+	wlr_keyboard_group_destroy(data);
+}
+
 void
 remove_keyboard(struct cg_seat *seat, struct cg_input_device *keyboard) {
 	if(!keyboard) {
@@ -443,12 +456,31 @@ remove_keyboard(struct cg_seat *seat, struct cg_input_device *keyboard) {
 	}
 	struct wlr_seat *wlr_seat = seat->seat;
 	struct wlr_keyboard *wlr_keyboard = keyboard->wlr_device->keyboard;
-	if(keyboard->wlr_device->keyboard->group) {
-		wlr_keyboard_group_remove_keyboard(wlr_keyboard->group, wlr_keyboard);
-	}
-	if(wlr_seat_get_keyboard(wlr_seat) == wlr_keyboard) {
-		wlr_seat_set_keyboard(wlr_seat, NULL);
+	struct wlr_keyboard_group *wlr_group = wlr_keyboard->group;
+	if(wlr_group) {
+		wlr_keyboard_group_remove_keyboard(wlr_group, wlr_keyboard);
+
+		if (wl_list_empty(&wlr_group->devices)) {
+			wlr_log(DEBUG, "Destroying empty keyboard group %p",
+					wlr_group);
+			struct cg_keyboard_group *group = wlr_group->data;
+			if(wlr_seat_get_keyboard(wlr_seat) == &wlr_group->keyboard) {
+				wlr_seat_set_keyboard(wlr_seat, NULL);
+			}
+			wlr_group->data = NULL;
+			wl_list_remove(&group->link);
+			wl_list_remove(&group->key.link);
+			wl_list_remove(&group->modifiers.link);
+			free(group);
+
+			// To prevent use-after-free conditions when handling key events, defer
+			// freeing the wlr_keyboard_group until idle
+			wl_event_loop_add_idle(seat->server->event_loop,
+					destroy_empty_wlr_keyboard_group, wlr_group);
+
+		}
 	}
+
 	--seat->num_keyboards;
 }