diff options
Diffstat (limited to 'bitbake/lib')
-rw-r--r-- | bitbake/lib/bb/__init__.py | 28 | ||||
-rw-r--r-- | bitbake/lib/bb/asyncrpc/serv.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/cooker.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/server/process.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/siggen.py | 6 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/fetch.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/support/httpserver.py | 4 | ||||
-rw-r--r-- | bitbake/lib/bb/utils.py | 20 | ||||
-rw-r--r-- | bitbake/lib/hashserv/tests.py | 2 |
9 files changed, 39 insertions, 29 deletions
diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py index bf4c54d829..3c98b835c6 100644 --- a/bitbake/lib/bb/__init__.py +++ b/bitbake/lib/bb/__init__.py | |||
@@ -37,6 +37,34 @@ class BBHandledException(Exception): | |||
37 | import os | 37 | import os |
38 | import logging | 38 | import logging |
39 | from collections import namedtuple | 39 | from collections import namedtuple |
40 | import multiprocessing as mp | ||
41 | |||
42 | # Python 3.14 changes the default multiprocessing context from "fork" to | ||
43 | # "forkserver". However, bitbake heavily relies on "fork" behavior to | ||
44 | # efficiently pass data to the child processes. Places that need this should do: | ||
45 | # from bb import multiprocessing | ||
46 | # in place of | ||
47 | # import multiprocessing | ||
48 | |||
49 | class MultiprocessingContext(object): | ||
50 | """ | ||
51 | Multiprocessing proxy object that uses the "fork" context for a property if | ||
52 | available, otherwise goes to the main multiprocessing module. This allows | ||
53 | it to be a drop-in replacement for the multiprocessing module, but use the | ||
54 | fork context | ||
55 | """ | ||
56 | def __init__(self): | ||
57 | super().__setattr__("_ctx", mp.get_context("fork")) | ||
58 | |||
59 | def __getattr__(self, name): | ||
60 | if hasattr(self._ctx, name): | ||
61 | return getattr(self._ctx, name) | ||
62 | return getattr(mp, name) | ||
63 | |||
64 | def __setattr__(self, name, value): | ||
65 | raise AttributeError(f"Unable to set attribute {name}") | ||
66 | |||
67 | multiprocessing = MultiprocessingContext() | ||
40 | 68 | ||
41 | 69 | ||
42 | class NullHandler(logging.Handler): | 70 | class NullHandler(logging.Handler): |
diff --git a/bitbake/lib/bb/asyncrpc/serv.py b/bitbake/lib/bb/asyncrpc/serv.py index 35e93f7c96..bd1aded8db 100644 --- a/bitbake/lib/bb/asyncrpc/serv.py +++ b/bitbake/lib/bb/asyncrpc/serv.py | |||
@@ -11,7 +11,7 @@ import os | |||
11 | import signal | 11 | import signal |
12 | import socket | 12 | import socket |
13 | import sys | 13 | import sys |
14 | import multiprocessing | 14 | from bb import multiprocessing |
15 | import logging | 15 | import logging |
16 | from .connection import StreamConnection, WebsocketConnection | 16 | from .connection import StreamConnection, WebsocketConnection |
17 | from .exceptions import ClientError, ServerError, ConnectionClosedError, InvokeError | 17 | from .exceptions import ClientError, ServerError, ConnectionClosedError, InvokeError |
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 0ad79bd53e..2eb64ef237 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -12,7 +12,7 @@ import enum | |||
12 | import sys, os, glob, os.path, re, time | 12 | import sys, os, glob, os.path, re, time |
13 | import itertools | 13 | import itertools |
14 | import logging | 14 | import logging |
15 | import multiprocessing | 15 | from bb import multiprocessing |
16 | import threading | 16 | import threading |
17 | from io import StringIO, UnsupportedOperation | 17 | from io import StringIO, UnsupportedOperation |
18 | from contextlib import closing | 18 | from contextlib import closing |
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py index 7d1b38f78c..d0f73590cc 100644 --- a/bitbake/lib/bb/server/process.py +++ b/bitbake/lib/bb/server/process.py | |||
@@ -13,7 +13,7 @@ | |||
13 | import bb | 13 | import bb |
14 | import bb.event | 14 | import bb.event |
15 | import logging | 15 | import logging |
16 | import multiprocessing | 16 | from bb import multiprocessing |
17 | import threading | 17 | import threading |
18 | import array | 18 | import array |
19 | import os | 19 | import os |
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index a6163b55ea..41eb643012 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py | |||
@@ -708,7 +708,7 @@ class SignatureGeneratorUniHashMixIn(object): | |||
708 | try: | 708 | try: |
709 | with self.client() as client: | 709 | with self.client() as client: |
710 | unihashes = client.get_unihash_batch((self._get_method(tid), self.taskhash[tid]) for tid in query_tids) | 710 | unihashes = client.get_unihash_batch((self._get_method(tid), self.taskhash[tid]) for tid in query_tids) |
711 | except (ConnectionError, FileNotFoundError) as e: | 711 | except (ConnectionError, FileNotFoundError, EOFError) as e: |
712 | bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) | 712 | bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) |
713 | 713 | ||
714 | for idx, tid in enumerate(query_tids): | 714 | for idx, tid in enumerate(query_tids): |
@@ -817,7 +817,7 @@ class SignatureGeneratorUniHashMixIn(object): | |||
817 | d.setVar('BB_UNIHASH', new_unihash) | 817 | d.setVar('BB_UNIHASH', new_unihash) |
818 | else: | 818 | else: |
819 | hashequiv_logger.debug('Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server)) | 819 | hashequiv_logger.debug('Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server)) |
820 | except (ConnectionError, FileNotFoundError) as e: | 820 | except (ConnectionError, FileNotFoundError, EOFError) as e: |
821 | bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) | 821 | bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) |
822 | finally: | 822 | finally: |
823 | if sigfile: | 823 | if sigfile: |
@@ -859,7 +859,7 @@ class SignatureGeneratorUniHashMixIn(object): | |||
859 | # TODO: What to do here? | 859 | # TODO: What to do here? |
860 | hashequiv_logger.verbose('Task %s unihash reported as unwanted hash %s' % (tid, finalunihash)) | 860 | hashequiv_logger.verbose('Task %s unihash reported as unwanted hash %s' % (tid, finalunihash)) |
861 | 861 | ||
862 | except (ConnectionError, FileNotFoundError) as e: | 862 | except (ConnectionError, FileNotFoundError, EOFError) as e: |
863 | bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) | 863 | bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) |
864 | 864 | ||
865 | return False | 865 | return False |
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py index 077472b8b3..cde1bf3390 100644 --- a/bitbake/lib/bb/tests/fetch.py +++ b/bitbake/lib/bb/tests/fetch.py | |||
@@ -1465,7 +1465,7 @@ class FetchLatestVersionTest(FetcherTest): | |||
1465 | # combination version pattern | 1465 | # combination version pattern |
1466 | ("sysprof", "git://git.yoctoproject.org/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "") | 1466 | ("sysprof", "git://git.yoctoproject.org/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "") |
1467 | : "1.2.0", | 1467 | : "1.2.0", |
1468 | ("u-boot-mkimage", "git://source.denx.de/u-boot/u-boot.git;branch=master;protocol=https", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "") | 1468 | ("u-boot-mkimage", "git://git.yoctoproject.org/bbfetchtests-u-boot.git;branch=master;protocol=https", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "") |
1469 | : "2014.01", | 1469 | : "2014.01", |
1470 | # version pattern "yyyymmdd" | 1470 | # version pattern "yyyymmdd" |
1471 | ("mobile-broadband-provider-info", "git://git.yoctoproject.org/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "") | 1471 | ("mobile-broadband-provider-info", "git://git.yoctoproject.org/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "") |
diff --git a/bitbake/lib/bb/tests/support/httpserver.py b/bitbake/lib/bb/tests/support/httpserver.py index 78f7660053..03327e923b 100644 --- a/bitbake/lib/bb/tests/support/httpserver.py +++ b/bitbake/lib/bb/tests/support/httpserver.py | |||
@@ -3,7 +3,7 @@ | |||
3 | # | 3 | # |
4 | 4 | ||
5 | import http.server | 5 | import http.server |
6 | import multiprocessing | 6 | from bb import multiprocessing |
7 | import os | 7 | import os |
8 | import traceback | 8 | import traceback |
9 | import signal | 9 | import signal |
@@ -43,7 +43,7 @@ class HTTPService(object): | |||
43 | self.process = multiprocessing.Process(target=self.server.server_start, args=[self.root_dir, self.logger]) | 43 | self.process = multiprocessing.Process(target=self.server.server_start, args=[self.root_dir, self.logger]) |
44 | 44 | ||
45 | # The signal handler from testimage.bbclass can cause deadlocks here | 45 | # The signal handler from testimage.bbclass can cause deadlocks here |
46 | # if the HTTPServer is terminated before it can restore the standard | 46 | # if the HTTPServer is terminated before it can restore the standard |
47 | #signal behaviour | 47 | #signal behaviour |
48 | orig = signal.getsignal(signal.SIGTERM) | 48 | orig = signal.getsignal(signal.SIGTERM) |
49 | signal.signal(signal.SIGTERM, signal.SIG_DFL) | 49 | signal.signal(signal.SIGTERM, signal.SIG_DFL) |
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py index c288c826c0..0cf3b1814c 100644 --- a/bitbake/lib/bb/utils.py +++ b/bitbake/lib/bb/utils.py | |||
@@ -12,7 +12,7 @@ import sys | |||
12 | import errno | 12 | import errno |
13 | import logging | 13 | import logging |
14 | import locale | 14 | import locale |
15 | import multiprocessing | 15 | from bb import multiprocessing |
16 | import importlib | 16 | import importlib |
17 | import importlib.machinery | 17 | import importlib.machinery |
18 | import importlib.util | 18 | import importlib.util |
@@ -1479,24 +1479,6 @@ def process_profilelog(fn, fn_out = None): | |||
1479 | p.print_callers() | 1479 | p.print_callers() |
1480 | 1480 | ||
1481 | 1481 | ||
1482 | # | ||
1483 | # Was present to work around multiprocessing pool bugs in python < 2.7.3 | ||
1484 | # | ||
1485 | def multiprocessingpool(*args, **kwargs): | ||
1486 | |||
1487 | import multiprocessing.pool | ||
1488 | #import multiprocessing.util | ||
1489 | #multiprocessing.util.log_to_stderr(10) | ||
1490 | # Deal with a multiprocessing bug where signals to the processes would be delayed until the work | ||
1491 | # completes. Putting in a timeout means the signals (like SIGINT/SIGTERM) get processed. | ||
1492 | def wrapper(func): | ||
1493 | def wrap(self, timeout=None): | ||
1494 | return func(self, timeout=timeout if timeout is not None else 1e100) | ||
1495 | return wrap | ||
1496 | multiprocessing.pool.IMapIterator.next = wrapper(multiprocessing.pool.IMapIterator.next) | ||
1497 | |||
1498 | return multiprocessing.Pool(*args, **kwargs) | ||
1499 | |||
1500 | def exec_flat_python_func(func, *args, **kwargs): | 1482 | def exec_flat_python_func(func, *args, **kwargs): |
1501 | """Execute a flat python function (defined with ``def funcname(args): ...``) | 1483 | """Execute a flat python function (defined with ``def funcname(args): ...``) |
1502 | 1484 | ||
diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py index da3f8e0884..124d8aa005 100644 --- a/bitbake/lib/hashserv/tests.py +++ b/bitbake/lib/hashserv/tests.py | |||
@@ -10,7 +10,7 @@ from .server import DEFAULT_ANON_PERMS, ALL_PERMISSIONS | |||
10 | from bb.asyncrpc import InvokeError | 10 | from bb.asyncrpc import InvokeError |
11 | import hashlib | 11 | import hashlib |
12 | import logging | 12 | import logging |
13 | import multiprocessing | 13 | from bb import multiprocessing |
14 | import os | 14 | import os |
15 | import sys | 15 | import sys |
16 | import tempfile | 16 | import tempfile |