From 1e01a7444536b4865feb94c398b68a936a463ddc Mon Sep 17 00:00:00 2001 From: Theodore Dubois Date: Tue, 17 Dec 2019 17:51:29 -0800 Subject: Port _FileDescriptorStreamsNonBlocking to use poll() select() has a limit of FD_SETSIZE file descriptors. If you run repo sync -j500 you'll pretty quickly hit this limit and get "file descriptor out of range for select" errors. poll() has no such limit. Change-Id: I21f350e472bda1db03dcbcc437645c23dbc7a901 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/248852 Reviewed-by: Mike Frysinger Tested-by: Theodore Dubois --- platform_utils.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'platform_utils.py') diff --git a/platform_utils.py b/platform_utils.py index 6684ebc8..0b139dbc 100644 --- a/platform_utils.py +++ b/platform_utils.py @@ -90,6 +90,11 @@ class _FileDescriptorStreamsNonBlocking(FileDescriptorStreams): """ Implementation of FileDescriptorStreams for platforms that support non blocking I/O. """ + def __init__(self): + super(_FileDescriptorStreamsNonBlocking, self).__init__() + self._poll = select.poll() + self._fd_to_stream = {} + class Stream(object): """ Encapsulates a file descriptor """ def __init__(self, fd, dest, std_name): @@ -113,11 +118,18 @@ class _FileDescriptorStreamsNonBlocking(FileDescriptorStreams): self.fd.close() def _create_stream(self, fd, dest, std_name): - return self.Stream(fd, dest, std_name) + stream = self.Stream(fd, dest, std_name) + self._fd_to_stream[stream.fileno()] = stream + self._poll.register(stream, select.POLLIN) + return stream + + def remove(self, stream): + self._poll.unregister(stream) + del self._fd_to_stream[stream.fileno()] + super(_FileDescriptorStreamsNonBlocking, self).remove(stream) def select(self): - ready_streams, _, _ = select.select(self.streams, [], []) - return ready_streams + return [self._fd_to_stream[fd] for fd, _ in self._poll.poll()] class _FileDescriptorStreamsThreads(FileDescriptorStreams): -- cgit v1.2.3-54-g00ecf