Work around Python 2.7 urllib2 bug

If the remote is using authenticated HTTP, but does not have
$GIT_URL/clone.bundle files in each repository, an initial sync
would fail around 8 projects in due to the library not resetting
the number of failures after getting a 404.

Work around this by updating the retry counter ourselves.

The urllib2 library is also not thread-safe. Make it somewhat
safer by wrapping the critical section with a lock.

Change-Id: I886e2750ef4793cbe2150c3b5396eb9f10974f7f
Signed-off-by: Shawn O. Pearce <sop@google.com>

2 files changed, 54 insertions, 34 deletions

diff --git a/main.py b/main.py
index 8ffdfcce..22e6fa42 100755
--- a/main.py
+++ b/main.py
@@ -273,6 +273,15 @@ class _UserAgentHandler(urllib2.BaseHandler):
     req.add_header('User-Agent', _UserAgent())
     return req
 
+class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+  def http_error_auth_reqed(self, authreq, host, req, headers):
+    try:
+      return urllib2.AbstractBasicAuthHandler.http_error_auth_reqed(
+        self, authreq, host, req, headers)
+    except:
+      self.reset_retry_count()
+      raise
+
 def init_http():
   handlers = [_UserAgentHandler()]
 
@@ -287,7 +296,7 @@ def init_http():
     pass
   except IOError:
     pass
-  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+  handlers.append(_BasicAuthHandler(mgr))
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']
diff --git a/project.py b/project.py
index 5f8369d5..4bc54de9 100644
--- a/project.py
+++ b/project.py
@@ -24,6 +24,11 @@ import sys
 import time
 import urllib2
 
+try:
+  import threading as _threading
+except ImportError:
+  import dummy_threading as _threading
+
 from color import Coloring
 from git_command import GitCommand
 from git_config import GitConfig, IsId, GetSchemeFromUrl
@@ -34,6 +39,8 @@ from progress import Progress
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
+_urllib_lock = _threading.Lock()
+
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
@@ -1458,40 +1465,44 @@ class Project(object):
       dest.seek(0, os.SEEK_END)
       pos = dest.tell()
 
-      req = urllib2.Request(srcUrl)
-      if pos > 0:
-        req.add_header('Range', 'bytes=%d-' % pos)
-
+      _urllib_lock.acquire()
       try:
-        r = urllib2.urlopen(req)
-      except urllib2.HTTPError, e:
-        def _content_type():
-          try:
-            return e.info()['content-type']
-          except:
-            return None
-
-        if e.code == 404:
-          keep = False
-          return False
-        elif _content_type() == 'text/plain':
-          try:
-            msg = e.read()
-            if len(msg) > 0 and msg[-1] == '\n':
-              msg = msg[0:-1]
-            msg = ' (%s)' % msg
-          except:
-            msg = ''
-        else:
-          try:
-            from BaseHTTPServer import BaseHTTPRequestHandler
-            res = BaseHTTPRequestHandler.responses[e.code]
-            msg = ' (%s: %s)' % (res[0], res[1])
-          except:
-            msg = ''
-        raise DownloadError('HTTP %s%s' % (e.code, msg))
-      except urllib2.URLError, e:
-        raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
+        req = urllib2.Request(srcUrl)
+        if pos > 0:
+          req.add_header('Range', 'bytes=%d-' % pos)
+
+        try:
+          r = urllib2.urlopen(req)
+        except urllib2.HTTPError, e:
+          def _content_type():
+            try:
+              return e.info()['content-type']
+            except:
+              return None
+
+          if e.code == 404:
+            keep = False
+            return False
+          elif _content_type() == 'text/plain':
+            try:
+              msg = e.read()
+              if len(msg) > 0 and msg[-1] == '\n':
+                msg = msg[0:-1]
+              msg = ' (%s)' % msg
+            except:
+              msg = ''
+          else:
+            try:
+              from BaseHTTPServer import BaseHTTPRequestHandler
+              res = BaseHTTPRequestHandler.responses[e.code]
+              msg = ' (%s: %s)' % (res[0], res[1])
+            except:
+              msg = ''
+          raise DownloadError('HTTP %s%s' % (e.code, msg))
+        except urllib2.URLError, e:
+          raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
+      finally:
+        _urllib_lock.release()
 
       p = None
       try: