--- a/src/evdev-lid.c
+++ b/src/evdev-lid.c
@@ -39,6 +39,7 @@
 	struct {
 		struct evdev_device *keyboard;
 		struct libinput_event_listener listener;
+		bool keyboard_is_monitored;
 	} keyboard;
 };
 
@@ -104,17 +105,22 @@
 	if (!dispatch->keyboard.keyboard)
 		return;
 
+	if (dispatch->keyboard.keyboard_is_monitored) {
+		libinput_device_remove_event_listener(
+					&dispatch->keyboard.listener);
+		libinput_device_init_event_listener(
+					&dispatch->keyboard.listener);
+
+		dispatch->keyboard.keyboard_is_monitored = false;
+	}
+
 	if (is_closed) {
 		libinput_device_add_event_listener(
 					&dispatch->keyboard.keyboard->base,
 					&dispatch->keyboard.listener,
 					lid_switch_keyboard_event,
 					dispatch);
-	} else {
-		libinput_device_remove_event_listener(
-					&dispatch->keyboard.listener);
-		libinput_device_init_event_listener(
-					&dispatch->keyboard.listener);
+		dispatch->keyboard.keyboard_is_monitored = true;
 	}
 }
 
@@ -193,7 +199,10 @@
 	if (!dispatch->keyboard.keyboard)
 		return;
 
-	libinput_device_remove_event_listener(&dispatch->keyboard.listener);
+	if (dispatch->keyboard.keyboard_is_monitored) {
+		libinput_device_remove_event_listener(&dispatch->keyboard.listener);
+		dispatch->keyboard.keyboard_is_monitored = false;
+	}
 }
 
 static void
@@ -246,10 +255,13 @@
 	struct lid_switch_dispatch *dispatch = lid_dispatch(device->dispatch);
 
 	if (removed_device == dispatch->keyboard.keyboard) {
-		libinput_device_remove_event_listener(
-					&dispatch->keyboard.listener);
-		libinput_device_init_event_listener(
-					&dispatch->keyboard.listener);
+		if (dispatch->keyboard.keyboard_is_monitored) {
+			libinput_device_remove_event_listener(
+						&dispatch->keyboard.listener);
+			libinput_device_init_event_listener(
+						&dispatch->keyboard.listener);
+			dispatch->keyboard.keyboard_is_monitored = false;
+		}
 		dispatch->keyboard.keyboard = NULL;
 	}
 }