Refactor downloading code into download.mk Support multiple file downloads Support svn downloads

SVN-Revision: 9057

7 files changed, 130 insertions, 34 deletions

diff --git a/include/download.mk b/include/download.mk
new file mode 100644
index 0000000000..1f227e221c
--- /dev/null
+++ b/include/download.mk
@@ -0,0 +1,90 @@
+# 
+# Copyright (C) 2007 OpenWrt.org
+#
+# This is free software, licensed under the GNU General Public License v2.
+# See /LICENSE for more information.
+#
+
+DOWNLOAD_RDEP:=$(STAMP_PREPARED)
+
+# Try to guess the download method from the URL
+define dl_method 
+$(strip \
+  $(if $(2),$(2), \
+    $(if $(filter @GNU/% @KERNEL/% @SF/% ftp://% http://%,$(1)),default, \
+      $(if $(filter git://%,$(1)),git, \
+        $(if $(filter svn://%,$(1)),svn, \
+          unknown \
+        ) \
+      ) \
+    ) \
+  ) \
+)
+endef
+
+# code for creating tarballs from svn/git checkouts - useful for mirror support
+dl_pack/bz2=tar cfj $(1) $(2)
+dl_pack/gz=tar cfz $(1) $(2)
+dl_pack/unknown=echo "ERROR: Unknown pack format for file $(1)"; false
+define dl_pack
+	$(if $(dl_pack/$(call ext,$(1))),$(dl_pack/$(call ext,$(1))),$(dl_pack/unknown))
+endef
+
+define DownloadMethod/unknown
+	@echo "ERROR: No download method available"; false
+endef
+
+define DownloadMethod/default
+	$(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(FILE)" "$(MD5SUM)" $(URL)
+endef
+
+define wrap_mirror
+	@$(if $(CONFIG_LOCALMIRROR),$(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(FILE)" "x" || ) \
+	( $(1) ) \
+	$(if $(CONFIG_LOCALMIRROR),, || $(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(FILE)" "x")
+endef
+
+define DownloadMethod/svn
+	$(call wrap_mirror, \
+		echo "Checking out files from svn repository..."; \
+		mkdir -p $(TMP_DIR)/dl && \
+		cd $(TMP_DIR)/dl && \
+		rm -rf $(SUBDIR) && \
+		[ \! -d $(SUBDIR) ] && \
+		svn co -r$(VERSION) $(URL) $(SUBDIR) && \
+		find $(SUBDIR) -name .svn | xargs rm -rf && \
+		echo "Packing checkout..." && \
+		$(call dl_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \
+		mv $(TMP_DIR)/dl/$(FILE) $(DL_DIR)/; \
+	)
+endef
+
+Validate/svn=VERSION SUBDIR
+#Validate/git=VERSION SUBDIR
+
+define Download/Defaults
+  URL:=
+  FILE:=
+  PROTO:=
+  MD5SUM:=
+  SUBDIR:=
+  VERSION:=
+endef
+
+define Download
+  $(eval $(Download/Defaults))
+  $(eval $(Download/$(1)))
+  $(foreach FIELD,URL FILE $(Validate/$(call dl_method,$(URL),$(PROTO))),
+    ifeq ($($(FIELD)),)
+      $$(error Download/$(1) is missing the $(FIELD) field.)
+    endif
+  )
+
+  $(if $(DOWNLOAD_RDEP),$(DOWNLOAD_RDEP): $(DL_DIR)/$(FILE))
+  download: $(DL_DIR)/$(FILE)
+
+  $(DL_DIR)/$(FILE):
+	mkdir -p $(DL_DIR)
+	$(if $(DownloadMethod/$(call dl_method,$(URL),$(PROTO))),$(DownloadMethod/$(call dl_method,$(URL),$(PROTO))),$(DownloadMethod/unknown))
+
+endef
diff --git a/include/host-build.mk b/include/host-build.mk
index 88129aad07..c0d62148e8 100644
--- a/include/host-build.mk
+++ b/include/host-build.mk
@@ -19,6 +19,7 @@ STAMP_INSTALLED:=$(STAGING_DIR_HOST)/stamp/.$(PKG_NAME)_installed
 
 override MAKEFLAGS=
 
+include $(INCLUDE_DIR)/download.mk
 include $(INCLUDE_DIR)/quilt.mk
 
 Build/Patch:=$(Build/Patch/Default)
@@ -69,17 +70,6 @@ define Build/Compile
   $(call Build/Compile/Default)
 endef
 
-		
-ifneq ($(strip $(PKG_SOURCE)),)
-  download: $(DL_DIR)/$(PKG_SOURCE)
-
-  $(DL_DIR)/$(PKG_SOURCE):
-	mkdir -p $(DL_DIR)
-	$(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(PKG_SOURCE)" "$(PKG_MD5SUM)" $(PKG_SOURCE_URL)
-
-  $(STAMP_PREPARED): $(DL_DIR)/$(PKG_SOURCE)
-endif
-
 ifneq ($(if $(QUILT),,$(CONFIG_AUTOREBUILD)),)
   define HostBuild/Autoclean
     $(call rdep,${CURDIR} $(PKG_FILE_DEPEND),$(STAMP_PREPARED))
@@ -87,10 +77,17 @@ ifneq ($(if $(QUILT),,$(CONFIG_AUTOREBUILD)),)
   endef
 endif
 
+define Download/default
+  FILE:=$(PKG_SOURCE)
+  URL:=$(PKG_SOURCE_URL)
+  PROTO:=$(PKG_SOURCE_PROTO)
+  VERSION:=$(PKG_SOURCE_VERSION)
+  MD5SUM:=$(PKG_MD5SUM)
+endef
+
 define HostBuild
-  ifeq ($(DUMP),)
-    $(call HostBuild/Autoclean)
-  endif
+  $(if $(strip $(PKG_SOURCE_URL)),$(call Download,default))
+  $(if $(DUMP),,$(call HostBuild/Autoclean))
   
   $(STAMP_PREPARED):
 	@-rm -rf $(PKG_BUILD_DIR)
diff --git a/include/kernel-build.mk b/include/kernel-build.mk
index 3b767c382b..82b0bcff27 100644
--- a/include/kernel-build.mk
+++ b/include/kernel-build.mk
@@ -17,6 +17,7 @@ endif
 
 STAMP_PREPARED:=$(LINUX_DIR)/.prepared
 STAMP_CONFIGURED:=$(LINUX_DIR)/.configured
+include $(INCLUDE_DIR)/download.mk
 include $(INCLUDE_DIR)/quilt.mk
 include $(INCLUDE_DIR)/kernel-defaults.mk
 
@@ -40,12 +41,14 @@ define Kernel/Clean
 	$(call Kernel/Clean/Default)
 endef
 
+define Download/kernel
+  URL:=$(LINUX_SITE)
+  FILE:=$(LINUX_SOURCE)
+  MD5SUM:=$(LINUX_KERNEL_MD5SUM)
+endef
+
 define BuildKernel
-  ifneq ($(LINUX_SITE),)
-    $(DL_DIR)/$(LINUX_SOURCE):
-		-mkdir -p $(DL_DIR)
-		$(SCRIPT_DIR)/download.pl $(DL_DIR) $(LINUX_SOURCE) $(LINUX_KERNEL_MD5SUM) $(LINUX_SITE)
-  endif
+  $(if $(LINUX_SITE),$(call Download,kernel))
 
   $(STAMP_PREPARED): $(DL_DIR)/$(LINUX_SOURCE)
 	-rm -rf $(KERNEL_BUILD_DIR)
diff --git a/include/package.mk b/include/package.mk
index 2b93972f24..bf82c60458 100644
--- a/include/package.mk
+++ b/include/package.mk
@@ -20,6 +20,7 @@ STAMP_PREPARED=$(PKG_BUILD_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call f
 STAMP_CONFIGURED:=$(PKG_BUILD_DIR)/.configured
 STAMP_BUILT:=$(PKG_BUILD_DIR)/.built
 
+include $(INCLUDE_DIR)/download.mk
 include $(INCLUDE_DIR)/quilt.mk
 include $(INCLUDE_DIR)/package-defaults.mk
 include $(INCLUDE_DIR)/package-dumpinfo.mk
@@ -38,18 +39,16 @@ ifeq ($(DUMP)$(filter prereq clean refresh update,$(MAKECMDGOALS)),)
   endif
 endif
 
+define Download/default
+  FILE:=$(PKG_SOURCE)
+  URL:=$(PKG_SOURCE_URL)
+  PROTO:=$(PKG_SOURCE_PROTO)
+  VERSION:=$(PKG_SOURCE_VERSION)
+  MD5SUM:=$(PKG_MD5SUM)
+endef
 
 define Build/DefaultTargets
-  ifneq ($(strip $(PKG_SOURCE_URL)),)
-    download: $(DL_DIR)/$(PKG_SOURCE)
-
-    $(DL_DIR)/$(PKG_SOURCE):
-	mkdir -p $(DL_DIR)
-	$(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(PKG_SOURCE)" "$(PKG_MD5SUM)" $(PKG_SOURCE_URL)
-
-    $(STAMP_PREPARED): $(DL_DIR)/$(PKG_SOURCE)
-  endif
-
+  $(if $(strip $(PKG_SOURCE_URL)),$(call Download,default))
   $(call Build/Autoclean)
 
   $(STAMP_PREPARED):
diff --git a/include/unpack.mk b/include/unpack.mk
index 500aada462..9b68e5dea3 100644
--- a/include/unpack.mk
+++ b/include/unpack.mk
@@ -1,4 +1,9 @@
-ext=$(word $(words $(subst ., ,$(1))),$(subst ., ,$(1)))
+# 
+# Copyright (C) 2006-2007 OpenWrt.org
+#
+# This is free software, licensed under the GNU General Public License v2.
+# See /LICENSE for more information.
+#
 
 # unpacking files with +s may break on some platforms. this typically emits error code 2
 ifneq ($(HOST_OS),Linux)
diff --git a/rules.mk b/rules.mk
index 9d5fcd72b4..9f1c4c4749 100644
--- a/rules.mk
+++ b/rules.mk
@@ -1,5 +1,5 @@
 # 
-# Copyright (C) 2006 OpenWrt.org
+# Copyright (C) 2006-2007 OpenWrt.org
 #
 # This is free software, licensed under the GNU General Public License v2.
 # See /LICENSE for more information.
@@ -132,6 +132,9 @@ $(call shvar,$(1))=$$(call $(1))
 export $(call shvar,$(1))
 endef
 
+# file extension
+ext=$(word $(words $(subst ., ,$(1))),$(subst ., ,$(1)))
+
 all:
 FORCE: ;
 .PHONY: FORCE
diff --git a/scripts/download.pl b/scripts/download.pl
index 66eaca8b36..161ae7f64b 100755
--- a/scripts/download.pl
+++ b/scripts/download.pl
@@ -10,16 +10,15 @@ use strict;
 use warnings;
 use File::Basename;
 
+@ARGV > 2 or die "Syntax: $0 <target dir> <filename> <md5sum> [<mirror> ...]\n";
+
 my $target = shift @ARGV;
 my $filename = shift @ARGV;
 my $md5sum = shift @ARGV;
 my $scriptdir = dirname($0);
 my @mirrors;
-
 my $ok;
 
-@ARGV > 0 or die "Syntax: $0 <target dir> <filename> <md5sum> <mirror> [<mirror> ...]\n";
-
 sub localmirrors {
     my @mlist;
     open LM, "$scriptdir/localmirrors" and do {