diff options
Diffstat (limited to 'bitbake/lib/bb/fetch/wget.py')
| -rw-r--r-- | bitbake/lib/bb/fetch/wget.py | 152 |
1 files changed, 42 insertions, 110 deletions
diff --git a/bitbake/lib/bb/fetch/wget.py b/bitbake/lib/bb/fetch/wget.py index e47a8859be..9c9c1675a1 100644 --- a/bitbake/lib/bb/fetch/wget.py +++ b/bitbake/lib/bb/fetch/wget.py | |||
| @@ -30,138 +30,70 @@ import bb | |||
| 30 | from bb import data | 30 | from bb import data |
| 31 | from bb.fetch import Fetch | 31 | from bb.fetch import Fetch |
| 32 | from bb.fetch import FetchError | 32 | from bb.fetch import FetchError |
| 33 | from bb.fetch import MD5SumError | ||
| 34 | from bb.fetch import uri_replace | 33 | from bb.fetch import uri_replace |
| 35 | 34 | ||
| 36 | class Wget(Fetch): | 35 | class Wget(Fetch): |
| 37 | """Class to fetch urls via 'wget'""" | 36 | """Class to fetch urls via 'wget'""" |
| 38 | def supports(url, d): | 37 | def supports(self, url, ud, d): |
| 39 | """Check to see if a given url can be fetched using wget. | ||
| 40 | Expects supplied url in list form, as outputted by bb.decodeurl(). | ||
| 41 | """ | 38 | """ |
| 42 | (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) | 39 | Check to see if a given url can be fetched with cvs. |
| 43 | return type in ['http','https','ftp'] | 40 | """ |
| 44 | supports = staticmethod(supports) | 41 | return ud.type in ['http','https','ftp'] |
| 45 | 42 | ||
| 46 | def localpath(url, d): | 43 | def localpath(self, url, ud, d): |
| 47 | # strip off parameters | 44 | |
| 48 | (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) | 45 | url = bb.encodeurl([ud.type, ud.host, ud.path, ud.user, ud.pswd, {}]) |
| 49 | if "localpath" in parm: | 46 | ud.basename = os.path.basename(ud.path) |
| 50 | # if user overrides local path, use it. | 47 | ud.localfile = data.expand(os.path.basename(url), d) |
| 51 | return parm["localpath"] | 48 | |
| 52 | url = bb.encodeurl([type, host, path, user, pswd, {}]) | 49 | return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) |
| 53 | 50 | ||
| 54 | return os.path.join(data.getVar("DL_DIR", d), os.path.basename(url)) | 51 | def go(self, uri, ud, d): |
| 55 | localpath = staticmethod(localpath) | ||
| 56 | |||
| 57 | def go(self, d, urls = []): | ||
| 58 | """Fetch urls""" | 52 | """Fetch urls""" |
| 59 | 53 | ||
| 60 | def md5_sum(parm, d): | 54 | def fetch_uri(uri, ud, d): |
| 61 | """ | 55 | if os.path.exists(ud.localpath): |
| 62 | Return the MD5SUM associated with the to be downloaded | 56 | # file exists, but we didnt complete it.. trying again.. |
| 63 | file. | ||
| 64 | It can return None if no md5sum is associated | ||
| 65 | """ | ||
| 66 | try: | ||
| 67 | return parm['md5sum'] | ||
| 68 | except: | ||
| 69 | return None | ||
| 70 | |||
| 71 | def verify_md5sum(wanted_sum, got_sum): | ||
| 72 | """ | ||
| 73 | Verify the md5sum we wanted with the one we got | ||
| 74 | """ | ||
| 75 | if not wanted_sum: | ||
| 76 | return True | ||
| 77 | |||
| 78 | return wanted_sum == got_sum | ||
| 79 | |||
| 80 | def fetch_uri(uri, basename, dl, md5, parm, d): | ||
| 81 | # the MD5 sum we want to verify | ||
| 82 | wanted_md5sum = md5_sum(parm, d) | ||
| 83 | if os.path.exists(dl): | ||
| 84 | # file exists, but we didnt complete it.. trying again.. | ||
| 85 | fetchcmd = data.getVar("RESUMECOMMAND", d, 1) | 57 | fetchcmd = data.getVar("RESUMECOMMAND", d, 1) |
| 86 | else: | 58 | else: |
| 87 | fetchcmd = data.getVar("FETCHCOMMAND", d, 1) | 59 | fetchcmd = data.getVar("FETCHCOMMAND", d, 1) |
| 88 | 60 | ||
| 89 | bb.note("fetch " + uri) | 61 | bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri) |
| 90 | fetchcmd = fetchcmd.replace("${URI}", uri) | 62 | fetchcmd = fetchcmd.replace("${URI}", uri) |
| 91 | fetchcmd = fetchcmd.replace("${FILE}", basename) | 63 | fetchcmd = fetchcmd.replace("${FILE}", ud.basename) |
| 92 | bb.debug(2, "executing " + fetchcmd) | 64 | bb.msg.debug(2, bb.msg.domain.Fetcher, "executing " + fetchcmd) |
| 93 | ret = os.system(fetchcmd) | 65 | ret = os.system(fetchcmd) |
| 94 | if ret != 0: | 66 | if ret != 0: |
| 95 | return False | 67 | return False |
| 96 | 68 | ||
| 97 | # check if sourceforge did send us to the mirror page | 69 | # check if sourceforge did send us to the mirror page |
| 98 | dl_dir = data.getVar("DL_DIR", d, True) | 70 | if not os.path.exists(ud.localpath): |
| 99 | if not os.path.exists(dl): | 71 | os.system("rm %s*" % ud.localpath) # FIXME shell quote it |
| 100 | os.system("rm %s*" % dl) # FIXME shell quote it | 72 | bb.msg.debug(2, bb.msg.domain.Fetcher, "sourceforge.net send us to the mirror on %s" % ud.basename) |
| 101 | bb.debug(2,"sourceforge.net send us to the mirror on %s" % basename) | ||
| 102 | return False | 73 | return False |
| 103 | 74 | ||
| 104 | # supposedly complete.. write out md5sum | ||
| 105 | if bb.which(data.getVar('PATH', d), 'md5sum'): | ||
| 106 | try: | ||
| 107 | md5pipe = os.popen('md5sum ' + dl) | ||
| 108 | md5data = (md5pipe.readline().split() or [ "" ])[0] | ||
| 109 | md5pipe.close() | ||
| 110 | except OSError: | ||
| 111 | md5data = "" | ||
| 112 | |||
| 113 | # verify the md5sum | ||
| 114 | if not verify_md5sum(wanted_md5sum, md5data): | ||
| 115 | raise MD5SumError(uri) | ||
| 116 | |||
| 117 | md5out = file(md5, 'w') | ||
| 118 | md5out.write(md5data) | ||
| 119 | md5out.close() | ||
| 120 | return True | 75 | return True |
| 121 | 76 | ||
| 122 | if not urls: | ||
| 123 | urls = self.urls | ||
| 124 | |||
| 125 | localdata = data.createCopy(d) | 77 | localdata = data.createCopy(d) |
| 126 | data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata) | 78 | data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata) |
| 127 | data.update_data(localdata) | 79 | data.update_data(localdata) |
| 128 | 80 | ||
| 129 | for uri in urls: | 81 | premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ] |
| 130 | completed = 0 | 82 | for (find, replace) in premirrors: |
| 131 | (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(uri, localdata)) | 83 | newuri = uri_replace(uri, find, replace, d) |
| 132 | basename = os.path.basename(path) | 84 | if newuri != uri: |
| 133 | dl = self.localpath(uri, d) | 85 | if fetch_uri(newuri, ud, localdata): |
| 134 | dl = data.expand(dl, localdata) | 86 | return |
| 135 | md5 = dl + '.md5' | 87 | |
| 136 | 88 | if fetch_uri(uri, ud, localdata): | |
| 137 | if os.path.exists(md5): | 89 | return |
| 138 | # complete, nothing to see here.. | 90 | |
| 139 | continue | 91 | # try mirrors |
| 140 | 92 | mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ] | |
| 141 | premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ] | 93 | for (find, replace) in mirrors: |
| 142 | for (find, replace) in premirrors: | 94 | newuri = uri_replace(uri, find, replace, d) |
| 143 | newuri = uri_replace(uri, find, replace, d) | 95 | if newuri != uri: |
| 144 | if newuri != uri: | 96 | if fetch_uri(newuri, ud, localdata): |
| 145 | if fetch_uri(newuri, basename, dl, md5, parm, localdata): | 97 | return |
| 146 | completed = 1 | 98 | |
| 147 | break | 99 | raise FetchError(uri) |
| 148 | |||
| 149 | if completed: | ||
| 150 | continue | ||
| 151 | |||
| 152 | if fetch_uri(uri, basename, dl, md5, parm, localdata): | ||
| 153 | continue | ||
| 154 | |||
| 155 | # try mirrors | ||
| 156 | mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ] | ||
| 157 | for (find, replace) in mirrors: | ||
| 158 | newuri = uri_replace(uri, find, replace, d) | ||
| 159 | if newuri != uri: | ||
| 160 | if fetch_uri(newuri, basename, dl, md5, parm, localdata): | ||
| 161 | completed = 1 | ||
| 162 | break | ||
| 163 | |||
| 164 | if not completed: | ||
| 165 | raise FetchError(uri) | ||
| 166 | |||
| 167 | del localdata | ||
