diff --git a/include/kernel-build.mk b/include/kernel-build.mk
index 432efd6ceb..cf8737c8ee 100644
--- a/include/kernel-build.mk
+++ b/include/kernel-build.mk
@@ -61,7 +61,7 @@ define BuildKernel
   $(if $(QUILT),$(Build/Quilt))
   $(if $(LINUX_SITE),$(call Download,kernel))
 
-  $(STAMP_PREPARED): $(DL_DIR)/$(LINUX_SOURCE)
+  $(STAMP_PREPARED): $(if $(LINUX_SITE),$(DL_DIR)/$(LINUX_SOURCE))
 	-rm -rf $(KERNEL_BUILD_DIR)
 	-mkdir -p $(KERNEL_BUILD_DIR)
 	$(Kernel/Prepare)