From cf31fe9b4fb650b27e19f5d7ee7297e383660caf Mon Sep 17 00:00:00 2001 From: The Android Open Source Project Date: Tue, 21 Oct 2008 07:00:00 -0700 Subject: Initial Contribution --- codereview/__init__.py | 1 + codereview/need_retry_pb2.py | 32 ++++ codereview/proto_client.py | 349 ++++++++++++++++++++++++++++++++++++++++ codereview/review_pb2.py | 48 ++++++ codereview/upload_bundle_pb2.py | 190 ++++++++++++++++++++++ 5 files changed, 620 insertions(+) create mode 100644 codereview/__init__.py create mode 100644 codereview/need_retry_pb2.py create mode 100755 codereview/proto_client.py create mode 100644 codereview/review_pb2.py create mode 100644 codereview/upload_bundle_pb2.py (limited to 'codereview') diff --git a/codereview/__init__.py b/codereview/__init__.py new file mode 100644 index 00000000..e47bc94e --- /dev/null +++ b/codereview/__init__.py @@ -0,0 +1 @@ +__version__ = 'v1.0' diff --git a/codereview/need_retry_pb2.py b/codereview/need_retry_pb2.py new file mode 100644 index 00000000..3fab2d43 --- /dev/null +++ b/codereview/need_retry_pb2.py @@ -0,0 +1,32 @@ +#!/usr/bin/python2.4 +# Generated by the protocol buffer compiler. DO NOT EDIT! + +from froofle.protobuf import descriptor +from froofle.protobuf import message +from froofle.protobuf import reflection +from froofle.protobuf import service +from froofle.protobuf import service_reflection +from froofle.protobuf import descriptor_pb2 + + + +_RETRYREQUESTLATERRESPONSE = descriptor.Descriptor( + name='RetryRequestLaterResponse', + full_name='codereview.RetryRequestLaterResponse', + filename='need_retry.proto', + containing_type=None, + fields=[ + ], + extensions=[ + ], + nested_types=[], # TODO(robinson): Implement. + enum_types=[ + ], + options=None) + + + +class RetryRequestLaterResponse(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + DESCRIPTOR = _RETRYREQUESTLATERRESPONSE + diff --git a/codereview/proto_client.py b/codereview/proto_client.py new file mode 100755 index 00000000..e11beff0 --- /dev/null +++ b/codereview/proto_client.py @@ -0,0 +1,349 @@ +# Copyright 2007, 2008 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import cookielib +import getpass +import logging +import md5 +import os +import random +import socket +import time +import urllib +import urllib2 +import urlparse + +from froofle.protobuf.service import RpcChannel +from froofle.protobuf.service import RpcController +from need_retry_pb2 import RetryRequestLaterResponse; + +class ClientLoginError(urllib2.HTTPError): + """Raised to indicate an error authenticating with ClientLogin.""" + + def __init__(self, url, code, msg, headers, args): + urllib2.HTTPError.__init__(self, url, code, msg, headers, None) + self.args = args + self.reason = args["Error"] + + +class Proxy(object): + class _ResultHolder(object): + def __call__(self, result): + self._result = result + + class _RemoteController(RpcController): + def Reset(self): + pass + + def Failed(self): + pass + + def ErrorText(self): + pass + + def StartCancel(self): + pass + + def SetFailed(self, reason): + raise RuntimeError, reason + + def IsCancelled(self): + pass + + def NotifyOnCancel(self, callback): + pass + + def __init__(self, stub): + self._stub = stub + + def __getattr__(self, key): + method = getattr(self._stub, key) + + def call(request): + done = self._ResultHolder() + method(self._RemoteController(), request, done) + return done._result + + return call + + +class HttpRpc(RpcChannel): + """Simple protobuf over HTTP POST implementation.""" + + def __init__(self, host, auth_function, + host_override=None, + extra_headers={}, + cookie_file=None): + """Creates a new HttpRpc. + + Args: + host: The host to send requests to. + auth_function: A function that takes no arguments and returns an + (email, password) tuple when called. Will be called if authentication + is required. + host_override: The host header to send to the server (defaults to host). + extra_headers: A dict of extra headers to append to every request. + cookie_file: If not None, name of the file in ~/ to save the + cookie jar into. Applications are encouraged to set this to + '.$appname_cookies' or some otherwise unique name. + """ + self.host = host.lower() + self.host_override = host_override + self.auth_function = auth_function + self.authenticated = False + self.extra_headers = extra_headers + self.xsrf_token = None + if cookie_file is None: + self.cookie_file = None + else: + self.cookie_file = os.path.expanduser("~/%s" % cookie_file) + self.opener = self._GetOpener() + if self.host_override: + logging.info("Server: %s; Host: %s", self.host, self.host_override) + else: + logging.info("Server: %s", self.host) + + def CallMethod(self, method, controller, request, response_type, done): + pat = "application/x-google-protobuf; name=%s" + + url = "/proto/%s/%s" % (method.containing_service.name, method.name) + reqbin = request.SerializeToString() + reqtyp = pat % request.DESCRIPTOR.full_name + reqmd5 = base64.b64encode(md5.new(reqbin).digest()) + + start = time.time() + while True: + t, b = self._Send(url, reqbin, reqtyp, reqmd5) + if t == (pat % RetryRequestLaterResponse.DESCRIPTOR.full_name): + if time.time() >= (start + 1800): + controller.SetFailed("timeout") + return + s = random.uniform(0.250, 2.000) + print "Busy, retrying in %.3f seconds ..." % s + time.sleep(s) + continue + + if t == (pat % response_type.DESCRIPTOR.full_name): + response = response_type() + response.ParseFromString(b) + done(response) + else: + controller.SetFailed("Unexpected %s response" % t) + break + + def _CreateRequest(self, url, data=None): + """Creates a new urllib request.""" + logging.debug("Creating request for: '%s' with payload:\n%s", url, data) + req = urllib2.Request(url, data=data) + if self.host_override: + req.add_header("Host", self.host_override) + for key, value in self.extra_headers.iteritems(): + req.add_header(key, value) + return req + + def _GetAuthToken(self, email, password): + """Uses ClientLogin to authenticate the user, returning an auth token. + + Args: + email: The user's email address + password: The user's password + + Raises: + ClientLoginError: If there was an error authenticating with ClientLogin. + HTTPError: If there was some other form of HTTP error. + + Returns: + The authentication token returned by ClientLogin. + """ + req = self._CreateRequest( + url="https://www.google.com/accounts/ClientLogin", + data=urllib.urlencode({ + "Email": email, + "Passwd": password, + "service": "ah", + "source": "gerrit-codereview-client", + "accountType": "HOSTED_OR_GOOGLE", + }) + ) + try: + response = self.opener.open(req) + response_body = response.read() + response_dict = dict(x.split("=") + for x in response_body.split("\n") if x) + return response_dict["Auth"] + except urllib2.HTTPError, e: + if e.code == 403: + body = e.read() + response_dict = dict(x.split("=", 1) for x in body.split("\n") if x) + raise ClientLoginError(req.get_full_url(), e.code, e.msg, + e.headers, response_dict) + else: + raise + + def _GetAuthCookie(self, auth_token): + """Fetches authentication cookies for an authentication token. + + Args: + auth_token: The authentication token returned by ClientLogin. + + Raises: + HTTPError: If there was an error fetching the authentication cookies. + """ + # This is a dummy value to allow us to identify when we're successful. + continue_location = "http://localhost/" + args = {"continue": continue_location, "auth": auth_token} + req = self._CreateRequest("http://%s/_ah/login?%s" % + (self.host, urllib.urlencode(args))) + try: + response = self.opener.open(req) + except urllib2.HTTPError, e: + response = e + if (response.code != 302 or + response.info()["location"] != continue_location): + raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg, + response.headers, response.fp) + self.authenticated = True + + def _GetXsrfToken(self): + """Fetches /proto/_token for use in X-XSRF-Token HTTP header. + + Raises: + HTTPError: If there was an error fetching a new token. + """ + tries = 0 + while True: + url = "http://%s/proto/_token" % self.host + req = self._CreateRequest(url) + try: + response = self.opener.open(req) + self.xsrf_token = response.read() + return + except urllib2.HTTPError, e: + if tries > 3: + raise + elif e.code == 401: + self._Authenticate() + else: + raise + + def _Authenticate(self): + """Authenticates the user. + + The authentication process works as follows: + 1) We get a username and password from the user + 2) We use ClientLogin to obtain an AUTH token for the user + (see http://code.google.com/apis/accounts/AuthForInstalledApps.html). + 3) We pass the auth token to /_ah/login on the server to obtain an + authentication cookie. If login was successful, it tries to redirect + us to the URL we provided. + + If we attempt to access the upload API without first obtaining an + authentication cookie, it returns a 401 response and directs us to + authenticate ourselves with ClientLogin. + """ + for i in range(3): + credentials = self.auth_function() + auth_token = self._GetAuthToken(credentials[0], credentials[1]) + self._GetAuthCookie(auth_token) + if self.cookie_file is not None: + self.cookie_jar.save() + return + + def _Send(self, request_path, payload, content_type, content_md5): + """Sends an RPC and returns the response. + + Args: + request_path: The path to send the request to, eg /api/appversion/create. + payload: The body of the request, or None to send an empty request. + content_type: The Content-Type header to use. + content_md5: The Content-MD5 header to use. + + Returns: + The content type, as a string. + The response body, as a string. + """ + if not self.authenticated: + self._Authenticate() + if not self.xsrf_token: + self._GetXsrfToken() + + old_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(None) + try: + tries = 0 + while True: + tries += 1 + url = "http://%s%s" % (self.host, request_path) + req = self._CreateRequest(url=url, data=payload) + req.add_header("Content-Type", content_type) + req.add_header("Content-MD5", content_md5) + req.add_header("X-XSRF-Token", self.xsrf_token) + try: + f = self.opener.open(req) + hdr = f.info() + type = hdr.getheader('Content-Type', + 'application/octet-stream') + response = f.read() + f.close() + return type, response + except urllib2.HTTPError, e: + if tries > 3: + raise + elif e.code == 401: + self._Authenticate() + elif e.code == 403: + if not hasattr(e, 'read'): + e.read = lambda self: '' + raise RuntimeError, '403\nxsrf: %s\n%s' \ + % (self.xsrf_token, e.read()) + else: + raise + finally: + socket.setdefaulttimeout(old_timeout) + + def _GetOpener(self): + """Returns an OpenerDirector that supports cookies and ignores redirects. + + Returns: + A urllib2.OpenerDirector object. + """ + opener = urllib2.OpenerDirector() + opener.add_handler(urllib2.ProxyHandler()) + opener.add_handler(urllib2.UnknownHandler()) + opener.add_handler(urllib2.HTTPHandler()) + opener.add_handler(urllib2.HTTPDefaultErrorHandler()) + opener.add_handler(urllib2.HTTPSHandler()) + opener.add_handler(urllib2.HTTPErrorProcessor()) + if self.cookie_file is not None: + self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file) + if os.path.exists(self.cookie_file): + try: + self.cookie_jar.load() + self.authenticated = True + except (cookielib.LoadError, IOError): + # Failed to load cookies - just ignore them. + pass + else: + # Create an empty cookie file with mode 600 + fd = os.open(self.cookie_file, os.O_CREAT, 0600) + os.close(fd) + # Always chmod the cookie file + os.chmod(self.cookie_file, 0600) + else: + # Don't save cookies across runs of update.py. + self.cookie_jar = cookielib.CookieJar() + opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) + return opener + diff --git a/codereview/review_pb2.py b/codereview/review_pb2.py new file mode 100644 index 00000000..0896feba --- /dev/null +++ b/codereview/review_pb2.py @@ -0,0 +1,48 @@ +#!/usr/bin/python2.4 +# Generated by the protocol buffer compiler. DO NOT EDIT! + +from froofle.protobuf import descriptor +from froofle.protobuf import message +from froofle.protobuf import reflection +from froofle.protobuf import service +from froofle.protobuf import service_reflection +from froofle.protobuf import descriptor_pb2 + + +import upload_bundle_pb2 + + + +_REVIEWSERVICE = descriptor.ServiceDescriptor( + name='ReviewService', + full_name='codereview.ReviewService', + index=0, + options=None, + methods=[ + descriptor.MethodDescriptor( + name='UploadBundle', + full_name='codereview.ReviewService.UploadBundle', + index=0, + containing_service=None, + input_type=upload_bundle_pb2._UPLOADBUNDLEREQUEST, + output_type=upload_bundle_pb2._UPLOADBUNDLERESPONSE, + options=None, + ), + descriptor.MethodDescriptor( + name='ContinueBundle', + full_name='codereview.ReviewService.ContinueBundle', + index=1, + containing_service=None, + input_type=upload_bundle_pb2._UPLOADBUNDLECONTINUE, + output_type=upload_bundle_pb2._UPLOADBUNDLERESPONSE, + options=None, + ), +]) + +class ReviewService(service.Service): + __metaclass__ = service_reflection.GeneratedServiceType + DESCRIPTOR = _REVIEWSERVICE +class ReviewService_Stub(ReviewService): + __metaclass__ = service_reflection.GeneratedServiceStubType + DESCRIPTOR = _REVIEWSERVICE + diff --git a/codereview/upload_bundle_pb2.py b/codereview/upload_bundle_pb2.py new file mode 100644 index 00000000..48c36512 --- /dev/null +++ b/codereview/upload_bundle_pb2.py @@ -0,0 +1,190 @@ +#!/usr/bin/python2.4 +# Generated by the protocol buffer compiler. DO NOT EDIT! + +from froofle.protobuf import descriptor +from froofle.protobuf import message +from froofle.protobuf import reflection +from froofle.protobuf import service +from froofle.protobuf import service_reflection +from froofle.protobuf import descriptor_pb2 + + +_UPLOADBUNDLERESPONSE_CODETYPE = descriptor.EnumDescriptor( + name='CodeType', + full_name='codereview.UploadBundleResponse.CodeType', + filename='CodeType', + values=[ + descriptor.EnumValueDescriptor( + name='RECEIVED', index=0, number=1, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='CONTINUE', index=1, number=4, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='UNAUTHORIZED_USER', index=2, number=7, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='UNKNOWN_PROJECT', index=3, number=2, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='UNKNOWN_BRANCH', index=4, number=3, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='UNKNOWN_BUNDLE', index=5, number=5, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='NOT_BUNDLE_OWNER', index=6, number=6, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='BUNDLE_CLOSED', index=7, number=8, + options=None, + type=None), + ], + options=None, +) + + +_UPLOADBUNDLEREQUEST = descriptor.Descriptor( + name='UploadBundleRequest', + full_name='codereview.UploadBundleRequest', + filename='upload_bundle.proto', + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='dest_project', full_name='codereview.UploadBundleRequest.dest_project', index=0, + number=10, type=9, cpp_type=9, label=2, + default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='dest_branch', full_name='codereview.UploadBundleRequest.dest_branch', index=1, + number=11, type=9, cpp_type=9, label=2, + default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='partial_upload', full_name='codereview.UploadBundleRequest.partial_upload', index=2, + number=12, type=8, cpp_type=7, label=2, + default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='bundle_data', full_name='codereview.UploadBundleRequest.bundle_data', index=3, + number=13, type=12, cpp_type=9, label=2, + default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='contained_object', full_name='codereview.UploadBundleRequest.contained_object', index=4, + number=1, type=9, cpp_type=9, label=3, + default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], # TODO(robinson): Implement. + enum_types=[ + ], + options=None) + + +_UPLOADBUNDLERESPONSE = descriptor.Descriptor( + name='UploadBundleResponse', + full_name='codereview.UploadBundleResponse', + filename='upload_bundle.proto', + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='status_code', full_name='codereview.UploadBundleResponse.status_code', index=0, + number=10, type=14, cpp_type=8, label=2, + default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='bundle_id', full_name='codereview.UploadBundleResponse.bundle_id', index=1, + number=11, type=9, cpp_type=9, label=1, + default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], # TODO(robinson): Implement. + enum_types=[ + _UPLOADBUNDLERESPONSE_CODETYPE, + ], + options=None) + + +_UPLOADBUNDLECONTINUE = descriptor.Descriptor( + name='UploadBundleContinue', + full_name='codereview.UploadBundleContinue', + filename='upload_bundle.proto', + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='bundle_id', full_name='codereview.UploadBundleContinue.bundle_id', index=0, + number=10, type=9, cpp_type=9, label=2, + default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='segment_id', full_name='codereview.UploadBundleContinue.segment_id', index=1, + number=11, type=5, cpp_type=1, label=2, + default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='partial_upload', full_name='codereview.UploadBundleContinue.partial_upload', index=2, + number=12, type=8, cpp_type=7, label=2, + default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='bundle_data', full_name='codereview.UploadBundleContinue.bundle_data', index=3, + number=13, type=12, cpp_type=9, label=1, + default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], # TODO(robinson): Implement. + enum_types=[ + ], + options=None) + + +_UPLOADBUNDLERESPONSE.fields_by_name['status_code'].enum_type = _UPLOADBUNDLERESPONSE_CODETYPE + +class UploadBundleRequest(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + DESCRIPTOR = _UPLOADBUNDLEREQUEST + +class UploadBundleResponse(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + DESCRIPTOR = _UPLOADBUNDLERESPONSE + +class UploadBundleContinue(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + DESCRIPTOR = _UPLOADBUNDLECONTINUE + -- cgit v1.2.3-54-g00ecf