diff options
-rw-r--r-- | bitbake/lib/bb/siggen.py | 53 |
1 files changed, 15 insertions, 38 deletions
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index 65ca0811d5..79f347db30 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py | |||
@@ -540,7 +540,7 @@ class SignatureGeneratorUniHashMixIn(object): | |||
540 | def __init__(self, data): | 540 | def __init__(self, data): |
541 | self.extramethod = {} | 541 | self.extramethod = {} |
542 | # NOTE: The cache only tracks hashes that exist. Hashes that don't | 542 | # NOTE: The cache only tracks hashes that exist. Hashes that don't |
543 | # exist are always queries from the server since it is possible for | 543 | # exist are always queried from the server since it is possible for |
544 | # hashes to appear over time, but much less likely for them to | 544 | # hashes to appear over time, but much less likely for them to |
545 | # disappear | 545 | # disappear |
546 | self.unihash_exists_cache = set() | 546 | self.unihash_exists_cache = set() |
@@ -558,11 +558,11 @@ class SignatureGeneratorUniHashMixIn(object): | |||
558 | super().__init__(data) | 558 | super().__init__(data) |
559 | 559 | ||
560 | def get_taskdata(self): | 560 | def get_taskdata(self): |
561 | return (self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env) + super().get_taskdata() | 561 | return (self.server, self.method, self.extramethod, self.username, self.password, self.env) + super().get_taskdata() |
562 | 562 | ||
563 | def set_taskdata(self, data): | 563 | def set_taskdata(self, data): |
564 | self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env = data[:7] | 564 | self.server, self.method, self.extramethod, self.username, self.password, self.env = data[:6] |
565 | super().set_taskdata(data[7:]) | 565 | super().set_taskdata(data[6:]) |
566 | 566 | ||
567 | def get_hashserv_creds(self): | 567 | def get_hashserv_creds(self): |
568 | if self.username and self.password: | 568 | if self.username and self.password: |
@@ -595,13 +595,6 @@ class SignatureGeneratorUniHashMixIn(object): | |||
595 | self._client = hashserv.create_client(self.server, **self.get_hashserv_creds()) | 595 | self._client = hashserv.create_client(self.server, **self.get_hashserv_creds()) |
596 | yield self._client | 596 | yield self._client |
597 | 597 | ||
598 | @contextmanager | ||
599 | def client_pool(self): | ||
600 | with self._client_env(): | ||
601 | if getattr(self, '_client_pool', None) is None: | ||
602 | self._client_pool = hashserv.client.ClientPool(self.server, self.max_parallel, **self.get_hashserv_creds()) | ||
603 | yield self._client_pool | ||
604 | |||
605 | def reset(self, data): | 598 | def reset(self, data): |
606 | self.__close_clients() | 599 | self.__close_clients() |
607 | return super().reset(data) | 600 | return super().reset(data) |
@@ -686,15 +679,10 @@ class SignatureGeneratorUniHashMixIn(object): | |||
686 | else: | 679 | else: |
687 | uncached_query[key] = unihash | 680 | uncached_query[key] = unihash |
688 | 681 | ||
689 | if self.max_parallel <= 1 or len(uncached_query) <= 1: | 682 | with self.client() as client: |
690 | # No parallelism required. Make the query serially with the single client | 683 | uncached_result = { |
691 | with self.client() as client: | 684 | key: client.unihash_exists(value) for key, value in uncached_query.items() |
692 | uncached_result = { | 685 | } |
693 | key: client.unihash_exists(value) for key, value in uncached_query.items() | ||
694 | } | ||
695 | else: | ||
696 | with self.client_pool() as client_pool: | ||
697 | uncached_result = client_pool.unihashes_exist(uncached_query) | ||
698 | 686 | ||
699 | for key, exists in uncached_result.items(): | 687 | for key, exists in uncached_result.items(): |
700 | if exists: | 688 | if exists: |
@@ -712,32 +700,20 @@ class SignatureGeneratorUniHashMixIn(object): | |||
712 | unihash | 700 | unihash |
713 | """ | 701 | """ |
714 | result = {} | 702 | result = {} |
715 | queries = {} | 703 | query_tids = [] |
716 | query_result = {} | ||
717 | 704 | ||
718 | for tid in tids: | 705 | for tid in tids: |
719 | unihash = self.get_cached_unihash(tid) | 706 | unihash = self.get_cached_unihash(tid) |
720 | if unihash: | 707 | if unihash: |
721 | result[tid] = unihash | 708 | result[tid] = unihash |
722 | else: | 709 | else: |
723 | queries[tid] = (self._get_method(tid), self.taskhash[tid]) | 710 | query_tids.append(tid) |
724 | |||
725 | if len(queries) == 0: | ||
726 | return result | ||
727 | 711 | ||
728 | if self.max_parallel <= 1 or len(queries) <= 1: | 712 | if query_tids: |
729 | # No parallelism required. Make the query using a single client | ||
730 | with self.client() as client: | 713 | with self.client() as client: |
731 | keys = list(queries.keys()) | 714 | unihashes = client.get_unihash_batch((self._get_method(tid), self.taskhash[tid]) for tid in query_tids) |
732 | unihashes = client.get_unihash_batch(queries[k] for k in keys) | ||
733 | 715 | ||
734 | for idx, k in enumerate(keys): | 716 | for idx, tid in enumerate(query_tids): |
735 | query_result[k] = unihashes[idx] | ||
736 | else: | ||
737 | with self.client_pool() as client_pool: | ||
738 | query_result = client_pool.get_unihashes(queries) | ||
739 | |||
740 | for tid, unihash in query_result.items(): | ||
741 | # In the absence of being able to discover a unique hash from the | 717 | # In the absence of being able to discover a unique hash from the |
742 | # server, make it be equivalent to the taskhash. The unique "hash" only | 718 | # server, make it be equivalent to the taskhash. The unique "hash" only |
743 | # really needs to be a unique string (not even necessarily a hash), but | 719 | # really needs to be a unique string (not even necessarily a hash), but |
@@ -752,6 +728,8 @@ class SignatureGeneratorUniHashMixIn(object): | |||
752 | # to the server, there is a better chance that they will agree on | 728 | # to the server, there is a better chance that they will agree on |
753 | # the unique hash. | 729 | # the unique hash. |
754 | taskhash = self.taskhash[tid] | 730 | taskhash = self.taskhash[tid] |
731 | unihash = unihashes[idx] | ||
732 | |||
755 | if unihash: | 733 | if unihash: |
756 | # A unique hash equal to the taskhash is not very interesting, | 734 | # A unique hash equal to the taskhash is not very interesting, |
757 | # so it is reported it at debug level 2. If they differ, that | 735 | # so it is reported it at debug level 2. If they differ, that |
@@ -898,7 +876,6 @@ class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureG | |||
898 | super().init_rundepcheck(data) | 876 | super().init_rundepcheck(data) |
899 | self.server = data.getVar('BB_HASHSERVE') | 877 | self.server = data.getVar('BB_HASHSERVE') |
900 | self.method = "sstate_output_hash" | 878 | self.method = "sstate_output_hash" |
901 | self.max_parallel = 1 | ||
902 | 879 | ||
903 | def clean_checksum_file_path(file_checksum_tuple): | 880 | def clean_checksum_file_path(file_checksum_tuple): |
904 | f, cs = file_checksum_tuple | 881 | f, cs = file_checksum_tuple |