summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--bitbake/lib/bb/siggen.py53
1 files changed, 15 insertions, 38 deletions
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 65ca0811d5..79f347db30 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -540,7 +540,7 @@ class SignatureGeneratorUniHashMixIn(object):
540 def __init__(self, data): 540 def __init__(self, data):
541 self.extramethod = {} 541 self.extramethod = {}
542 # NOTE: The cache only tracks hashes that exist. Hashes that don't 542 # NOTE: The cache only tracks hashes that exist. Hashes that don't
543 # exist are always queries from the server since it is possible for 543 # exist are always queried from the server since it is possible for
544 # hashes to appear over time, but much less likely for them to 544 # hashes to appear over time, but much less likely for them to
545 # disappear 545 # disappear
546 self.unihash_exists_cache = set() 546 self.unihash_exists_cache = set()
@@ -558,11 +558,11 @@ class SignatureGeneratorUniHashMixIn(object):
558 super().__init__(data) 558 super().__init__(data)
559 559
560 def get_taskdata(self): 560 def get_taskdata(self):
561 return (self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env) + super().get_taskdata() 561 return (self.server, self.method, self.extramethod, self.username, self.password, self.env) + super().get_taskdata()
562 562
563 def set_taskdata(self, data): 563 def set_taskdata(self, data):
564 self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env = data[:7] 564 self.server, self.method, self.extramethod, self.username, self.password, self.env = data[:6]
565 super().set_taskdata(data[7:]) 565 super().set_taskdata(data[6:])
566 566
567 def get_hashserv_creds(self): 567 def get_hashserv_creds(self):
568 if self.username and self.password: 568 if self.username and self.password:
@@ -595,13 +595,6 @@ class SignatureGeneratorUniHashMixIn(object):
595 self._client = hashserv.create_client(self.server, **self.get_hashserv_creds()) 595 self._client = hashserv.create_client(self.server, **self.get_hashserv_creds())
596 yield self._client 596 yield self._client
597 597
598 @contextmanager
599 def client_pool(self):
600 with self._client_env():
601 if getattr(self, '_client_pool', None) is None:
602 self._client_pool = hashserv.client.ClientPool(self.server, self.max_parallel, **self.get_hashserv_creds())
603 yield self._client_pool
604
605 def reset(self, data): 598 def reset(self, data):
606 self.__close_clients() 599 self.__close_clients()
607 return super().reset(data) 600 return super().reset(data)
@@ -686,15 +679,10 @@ class SignatureGeneratorUniHashMixIn(object):
686 else: 679 else:
687 uncached_query[key] = unihash 680 uncached_query[key] = unihash
688 681
689 if self.max_parallel <= 1 or len(uncached_query) <= 1: 682 with self.client() as client:
690 # No parallelism required. Make the query serially with the single client 683 uncached_result = {
691 with self.client() as client: 684 key: client.unihash_exists(value) for key, value in uncached_query.items()
692 uncached_result = { 685 }
693 key: client.unihash_exists(value) for key, value in uncached_query.items()
694 }
695 else:
696 with self.client_pool() as client_pool:
697 uncached_result = client_pool.unihashes_exist(uncached_query)
698 686
699 for key, exists in uncached_result.items(): 687 for key, exists in uncached_result.items():
700 if exists: 688 if exists:
@@ -712,32 +700,20 @@ class SignatureGeneratorUniHashMixIn(object):
712 unihash 700 unihash
713 """ 701 """
714 result = {} 702 result = {}
715 queries = {} 703 query_tids = []
716 query_result = {}
717 704
718 for tid in tids: 705 for tid in tids:
719 unihash = self.get_cached_unihash(tid) 706 unihash = self.get_cached_unihash(tid)
720 if unihash: 707 if unihash:
721 result[tid] = unihash 708 result[tid] = unihash
722 else: 709 else:
723 queries[tid] = (self._get_method(tid), self.taskhash[tid]) 710 query_tids.append(tid)
724
725 if len(queries) == 0:
726 return result
727 711
728 if self.max_parallel <= 1 or len(queries) <= 1: 712 if query_tids:
729 # No parallelism required. Make the query using a single client
730 with self.client() as client: 713 with self.client() as client:
731 keys = list(queries.keys()) 714 unihashes = client.get_unihash_batch((self._get_method(tid), self.taskhash[tid]) for tid in query_tids)
732 unihashes = client.get_unihash_batch(queries[k] for k in keys)
733 715
734 for idx, k in enumerate(keys): 716 for idx, tid in enumerate(query_tids):
735 query_result[k] = unihashes[idx]
736 else:
737 with self.client_pool() as client_pool:
738 query_result = client_pool.get_unihashes(queries)
739
740 for tid, unihash in query_result.items():
741 # In the absence of being able to discover a unique hash from the 717 # In the absence of being able to discover a unique hash from the
742 # server, make it be equivalent to the taskhash. The unique "hash" only 718 # server, make it be equivalent to the taskhash. The unique "hash" only
743 # really needs to be a unique string (not even necessarily a hash), but 719 # really needs to be a unique string (not even necessarily a hash), but
@@ -752,6 +728,8 @@ class SignatureGeneratorUniHashMixIn(object):
752 # to the server, there is a better chance that they will agree on 728 # to the server, there is a better chance that they will agree on
753 # the unique hash. 729 # the unique hash.
754 taskhash = self.taskhash[tid] 730 taskhash = self.taskhash[tid]
731 unihash = unihashes[idx]
732
755 if unihash: 733 if unihash:
756 # A unique hash equal to the taskhash is not very interesting, 734 # A unique hash equal to the taskhash is not very interesting,
757 # so it is reported it at debug level 2. If they differ, that 735 # so it is reported it at debug level 2. If they differ, that
@@ -898,7 +876,6 @@ class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureG
898 super().init_rundepcheck(data) 876 super().init_rundepcheck(data)
899 self.server = data.getVar('BB_HASHSERVE') 877 self.server = data.getVar('BB_HASHSERVE')
900 self.method = "sstate_output_hash" 878 self.method = "sstate_output_hash"
901 self.max_parallel = 1
902 879
903def clean_checksum_file_path(file_checksum_tuple): 880def clean_checksum_file_path(file_checksum_tuple):
904 f, cs = file_checksum_tuple 881 f, cs = file_checksum_tuple