diff --git a/firmware/Makefile b/firmware/Makefile
index e69461f..cc25f56 100644
--- a/firmware/Makefile
+++ b/firmware/Makefile
@@ -9,6 +9,11 @@
 
 fw-external-y := $(subst ",,$(CONFIG_EXTRA_FIRMWARE))
 
+# If CONFIG_FIRMWARE_IN_KERNEL is not set, then don't include any firmware
+ifneq ($(CONFIG_FIRMWARE_IN_KERNEL),y)
+fw-shipped-y :=
+endif
+
 firmware-y    := $(fw-external-y) $(fw-shipped-y)
 firmware-dirs := $(sort $(patsubst %,$(objtree)/$(obj)/%/,$(dir $(firmware-y) $(fw-shipped-))))