diff options
| -rw-r--r-- | meta/recipes-devtools/python/python-native_2.7.15.bb (renamed from meta/recipes-devtools/python/python-native_2.7.14.bb) | 1 | ||||
| -rw-r--r-- | meta/recipes-devtools/python/python.inc | 10 | ||||
| -rw-r--r-- | meta/recipes-devtools/python/python/CVE-2018-1000030-1.patch | 138 | ||||
| -rw-r--r-- | meta/recipes-devtools/python/python/CVE-2018-1000030-2.patch | 306 | ||||
| -rw-r--r-- | meta/recipes-devtools/python/python/fix-gc-alignment.patch | 43 | ||||
| -rw-r--r-- | meta/recipes-devtools/python/python_2.7.15.bb (renamed from meta/recipes-devtools/python/python_2.7.14.bb) | 1 |
6 files changed, 4 insertions, 495 deletions
diff --git a/meta/recipes-devtools/python/python-native_2.7.14.bb b/meta/recipes-devtools/python/python-native_2.7.15.bb index b822583c53..7c491fa3e0 100644 --- a/meta/recipes-devtools/python/python-native_2.7.14.bb +++ b/meta/recipes-devtools/python/python-native_2.7.15.bb | |||
| @@ -16,7 +16,6 @@ SRC_URI += "\ | |||
| 16 | file://builddir.patch \ | 16 | file://builddir.patch \ |
| 17 | file://parallel-makeinst-create-bindir.patch \ | 17 | file://parallel-makeinst-create-bindir.patch \ |
| 18 | file://revert_use_of_sysconfigdata.patch \ | 18 | file://revert_use_of_sysconfigdata.patch \ |
| 19 | file://fix-gc-alignment.patch \ | ||
| 20 | " | 19 | " |
| 21 | 20 | ||
| 22 | S = "${WORKDIR}/Python-${PV}" | 21 | S = "${WORKDIR}/Python-${PV}" |
diff --git a/meta/recipes-devtools/python/python.inc b/meta/recipes-devtools/python/python.inc index 69542c96c8..062a20f68e 100644 --- a/meta/recipes-devtools/python/python.inc +++ b/meta/recipes-devtools/python/python.inc | |||
| @@ -5,14 +5,12 @@ SECTION = "devel/python" | |||
| 5 | # bump this on every change in contrib/python/generate-manifest-2.7.py | 5 | # bump this on every change in contrib/python/generate-manifest-2.7.py |
| 6 | INC_PR = "r1" | 6 | INC_PR = "r1" |
| 7 | 7 | ||
| 8 | LIC_FILES_CHKSUM = "file://LICENSE;md5=f741e51de91d4eeea5930b9c3c7fa69d" | 8 | LIC_FILES_CHKSUM = "file://LICENSE;md5=f257cc14f81685691652a3d3e1b5d754" |
| 9 | 9 | ||
| 10 | SRC_URI = "http://www.python.org/ftp/python/${PV}/Python-${PV}.tar.xz \ | 10 | SRC_URI = "http://www.python.org/ftp/python/${PV}/Python-${PV}.tar.xz" |
| 11 | file://CVE-2018-1000030-1.patch \ | ||
| 12 | file://CVE-2018-1000030-2.patch" | ||
| 13 | 11 | ||
| 14 | SRC_URI[md5sum] = "1f6db41ad91d9eb0a6f0c769b8613c5b" | 12 | SRC_URI[md5sum] = "a80ae3cc478460b922242f43a1b4094d" |
| 15 | SRC_URI[sha256sum] = "71ffb26e09e78650e424929b2b457b9c912ac216576e6bd9e7d204ed03296a66" | 13 | SRC_URI[sha256sum] = "22d9b1ac5b26135ad2b8c2901a9413537e08749a753356ee913c84dbd2df5574" |
| 16 | 14 | ||
| 17 | # python recipe is actually python 2.x | 15 | # python recipe is actually python 2.x |
| 18 | # also, exclude pre-releases for both python 2.x and 3.x | 16 | # also, exclude pre-releases for both python 2.x and 3.x |
diff --git a/meta/recipes-devtools/python/python/CVE-2018-1000030-1.patch b/meta/recipes-devtools/python/python/CVE-2018-1000030-1.patch deleted file mode 100644 index 06ad4c695d..0000000000 --- a/meta/recipes-devtools/python/python/CVE-2018-1000030-1.patch +++ /dev/null | |||
| @@ -1,138 +0,0 @@ | |||
| 1 | From 6401e5671781eb217ee1afb4603cc0d1b0367ae6 Mon Sep 17 00:00:00 2001 | ||
| 2 | From: Serhiy Storchaka <storchaka@gmail.com> | ||
| 3 | Date: Fri, 10 Nov 2017 12:58:55 +0200 | ||
| 4 | Subject: [PATCH] [2.7] bpo-31530: Stop crashes when iterating over a file on | ||
| 5 | multiple threads. (#3672) | ||
| 6 | |||
| 7 | CVE: CVE-2018-1000030 | ||
| 8 | Upstream-Status: Backport [https://github.com/python/cpython/commit/6401e5671781eb217ee1afb4603cc0d1b0367ae6] | ||
| 9 | |||
| 10 | Signed-off-by: Jagadeesh Krishnanjanappa <jkrishnanjanappa@mvista.com> | ||
| 11 | --- | ||
| 12 | Lib/test/test_file2k.py | 32 ++++++++++++++++++++++ | ||
| 13 | .../2017-09-20-18-28-09.bpo-31530.CdLOM7.rst | 4 +++ | ||
| 14 | Objects/fileobject.c | 19 +++++++++++-- | ||
| 15 | 3 files changed, 52 insertions(+), 3 deletions(-) | ||
| 16 | create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst | ||
| 17 | |||
| 18 | diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py | ||
| 19 | index e39ef7042e..d8966e034e 100644 | ||
| 20 | --- a/Lib/test/test_file2k.py | ||
| 21 | +++ b/Lib/test/test_file2k.py | ||
| 22 | @@ -652,6 +652,38 @@ class FileThreadingTests(unittest.TestCase): | ||
| 23 | self.f.writelines('') | ||
| 24 | self._test_close_open_io(io_func) | ||
| 25 | |||
| 26 | + def test_iteration_torture(self): | ||
| 27 | + # bpo-31530: Crash when concurrently iterate over a file. | ||
| 28 | + with open(self.filename, "wb") as fp: | ||
| 29 | + for i in xrange(2**20): | ||
| 30 | + fp.write(b"0"*50 + b"\n") | ||
| 31 | + with open(self.filename, "rb") as f: | ||
| 32 | + def iterate(): | ||
| 33 | + try: | ||
| 34 | + for l in f: | ||
| 35 | + pass | ||
| 36 | + except IOError: | ||
| 37 | + pass | ||
| 38 | + self._run_workers(iterate, 10) | ||
| 39 | + | ||
| 40 | + def test_iteration_seek(self): | ||
| 41 | + # bpo-31530: Crash when concurrently seek and iterate over a file. | ||
| 42 | + with open(self.filename, "wb") as fp: | ||
| 43 | + for i in xrange(10000): | ||
| 44 | + fp.write(b"0"*50 + b"\n") | ||
| 45 | + with open(self.filename, "rb") as f: | ||
| 46 | + it = iter([1] + [0]*10) # one thread reads, others seek | ||
| 47 | + def iterate(): | ||
| 48 | + try: | ||
| 49 | + if next(it): | ||
| 50 | + for l in f: | ||
| 51 | + pass | ||
| 52 | + else: | ||
| 53 | + for i in range(100): | ||
| 54 | + f.seek(i*100, 0) | ||
| 55 | + except IOError: | ||
| 56 | + pass | ||
| 57 | + self._run_workers(iterate, 10) | ||
| 58 | |||
| 59 | @unittest.skipUnless(os.name == 'posix', 'test requires a posix system.') | ||
| 60 | class TestFileSignalEINTR(unittest.TestCase): | ||
| 61 | diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst | ||
| 62 | new file mode 100644 | ||
| 63 | index 0000000000..a6cb6c9e9b | ||
| 64 | --- /dev/null | ||
| 65 | +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst | ||
| 66 | @@ -0,0 +1,4 @@ | ||
| 67 | +Fixed crashes when iterating over a file on multiple threads. | ||
| 68 | +seek() and next() methods of file objects now raise an exception during | ||
| 69 | +concurrent operation on the same file object. | ||
| 70 | +A lock can be used to prevent the error. | ||
| 71 | diff --git a/Objects/fileobject.c b/Objects/fileobject.c | ||
| 72 | index 7e07a5376f..2f63c374d1 100644 | ||
| 73 | --- a/Objects/fileobject.c | ||
| 74 | +++ b/Objects/fileobject.c | ||
| 75 | @@ -430,7 +430,7 @@ close_the_file(PyFileObject *f) | ||
| 76 | if (f->ob_refcnt > 0) { | ||
| 77 | PyErr_SetString(PyExc_IOError, | ||
| 78 | "close() called during concurrent " | ||
| 79 | - "operation on the same file object."); | ||
| 80 | + "operation on the same file object"); | ||
| 81 | } else { | ||
| 82 | /* This should not happen unless someone is | ||
| 83 | * carelessly playing with the PyFileObject | ||
| 84 | @@ -438,7 +438,7 @@ close_the_file(PyFileObject *f) | ||
| 85 | * pointer. */ | ||
| 86 | PyErr_SetString(PyExc_SystemError, | ||
| 87 | "PyFileObject locking error in " | ||
| 88 | - "destructor (refcnt <= 0 at close)."); | ||
| 89 | + "destructor (refcnt <= 0 at close)"); | ||
| 90 | } | ||
| 91 | return NULL; | ||
| 92 | } | ||
| 93 | @@ -762,6 +762,12 @@ file_seek(PyFileObject *f, PyObject *args) | ||
| 94 | |||
| 95 | if (f->f_fp == NULL) | ||
| 96 | return err_closed(); | ||
| 97 | + if (f->unlocked_count > 0) { | ||
| 98 | + PyErr_SetString(PyExc_IOError, | ||
| 99 | + "seek() called during concurrent " | ||
| 100 | + "operation on the same file object"); | ||
| 101 | + return NULL; | ||
| 102 | + } | ||
| 103 | drop_readahead(f); | ||
| 104 | whence = 0; | ||
| 105 | if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &whence)) | ||
| 106 | @@ -2238,6 +2244,7 @@ readahead(PyFileObject *f, Py_ssize_t bufsize) | ||
| 107 | { | ||
| 108 | Py_ssize_t chunksize; | ||
| 109 | |||
| 110 | + assert(f->unlocked_count == 0); | ||
| 111 | if (f->f_buf != NULL) { | ||
| 112 | if( (f->f_bufend - f->f_bufptr) >= 1) | ||
| 113 | return 0; | ||
| 114 | @@ -2279,6 +2286,12 @@ readahead_get_line_skip(PyFileObject *f, Py_ssize_t skip, Py_ssize_t bufsize) | ||
| 115 | char *buf; | ||
| 116 | Py_ssize_t len; | ||
| 117 | |||
| 118 | + if (f->unlocked_count > 0) { | ||
| 119 | + PyErr_SetString(PyExc_IOError, | ||
| 120 | + "next() called during concurrent " | ||
| 121 | + "operation on the same file object"); | ||
| 122 | + return NULL; | ||
| 123 | + } | ||
| 124 | if (f->f_buf == NULL) | ||
| 125 | if (readahead(f, bufsize) < 0) | ||
| 126 | return NULL; | ||
| 127 | @@ -2692,7 +2705,7 @@ int PyObject_AsFileDescriptor(PyObject *o) | ||
| 128 | } | ||
| 129 | else { | ||
| 130 | PyErr_SetString(PyExc_TypeError, | ||
| 131 | - "argument must be an int, or have a fileno() method."); | ||
| 132 | + "argument must be an int, or have a fileno() method"); | ||
| 133 | return -1; | ||
| 134 | } | ||
| 135 | |||
| 136 | -- | ||
| 137 | 2.13.3 | ||
| 138 | |||
diff --git a/meta/recipes-devtools/python/python/CVE-2018-1000030-2.patch b/meta/recipes-devtools/python/python/CVE-2018-1000030-2.patch deleted file mode 100644 index 9b7713be88..0000000000 --- a/meta/recipes-devtools/python/python/CVE-2018-1000030-2.patch +++ /dev/null | |||
| @@ -1,306 +0,0 @@ | |||
| 1 | From dbf52e02f18dac6f5f0a64f78932f3dc6efc056b Mon Sep 17 00:00:00 2001 | ||
| 2 | From: Benjamin Peterson <benjamin@python.org> | ||
| 3 | Date: Tue, 2 Jan 2018 09:25:41 -0800 | ||
| 4 | Subject: [PATCH] bpo-31530: fix crash when multiple threads iterate over a | ||
| 5 | file, round 2 (#5060) | ||
| 6 | |||
| 7 | Multiple threads iterating over a file can corrupt the file's internal readahead | ||
| 8 | buffer resulting in crashes. To fix this, cache buffer state thread-locally for | ||
| 9 | the duration of a file_iternext call and only update the file's internal state | ||
| 10 | after reading completes. | ||
| 11 | |||
| 12 | No attempt is made to define or provide "reasonable" semantics for iterating | ||
| 13 | over a file on multiple threads. (Non-crashing) races are still | ||
| 14 | present. Duplicated, corrupt, and missing data will happen. | ||
| 15 | |||
| 16 | This was originally fixed by 6401e5671781eb217ee1afb4603cc0d1b0367ae6, which | ||
| 17 | raised an exception from seek() and next() when concurrent operations were | ||
| 18 | detected. Alas, this simpler solution breaks legitimate use cases such as | ||
| 19 | capturing the standard streams when multiple threads are logging. | ||
| 20 | |||
| 21 | CVE: CVE-2018-1000030 | ||
| 22 | Upstream-Status: Backport [https://github.com/python/cpython/commit/dbf52e02f18dac6f5f0a64f78932f3dc6efc056b] | ||
| 23 | |||
| 24 | Signed-off-by: Jagadeesh Krishnanjanappa <jkrishnanjanappa@mvista.com> | ||
| 25 | |||
| 26 | --- | ||
| 27 | Lib/test/test_file2k.py | 27 ++--- | ||
| 28 | .../2017-09-20-18-28-09.bpo-31530.CdLOM7.rst | 3 - | ||
| 29 | Objects/fileobject.c | 118 ++++++++++++--------- | ||
| 30 | 3 files changed, 78 insertions(+), 70 deletions(-) | ||
| 31 | |||
| 32 | diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py | ||
| 33 | index d8966e034e..c73e8d8dc4 100644 | ||
| 34 | --- a/Lib/test/test_file2k.py | ||
| 35 | +++ b/Lib/test/test_file2k.py | ||
| 36 | @@ -653,18 +653,15 @@ class FileThreadingTests(unittest.TestCase): | ||
| 37 | self._test_close_open_io(io_func) | ||
| 38 | |||
| 39 | def test_iteration_torture(self): | ||
| 40 | - # bpo-31530: Crash when concurrently iterate over a file. | ||
| 41 | + # bpo-31530 | ||
| 42 | with open(self.filename, "wb") as fp: | ||
| 43 | for i in xrange(2**20): | ||
| 44 | fp.write(b"0"*50 + b"\n") | ||
| 45 | with open(self.filename, "rb") as f: | ||
| 46 | - def iterate(): | ||
| 47 | - try: | ||
| 48 | - for l in f: | ||
| 49 | - pass | ||
| 50 | - except IOError: | ||
| 51 | + def it(): | ||
| 52 | + for l in f: | ||
| 53 | pass | ||
| 54 | - self._run_workers(iterate, 10) | ||
| 55 | + self._run_workers(it, 10) | ||
| 56 | |||
| 57 | def test_iteration_seek(self): | ||
| 58 | # bpo-31530: Crash when concurrently seek and iterate over a file. | ||
| 59 | @@ -674,17 +671,15 @@ class FileThreadingTests(unittest.TestCase): | ||
| 60 | with open(self.filename, "rb") as f: | ||
| 61 | it = iter([1] + [0]*10) # one thread reads, others seek | ||
| 62 | def iterate(): | ||
| 63 | - try: | ||
| 64 | - if next(it): | ||
| 65 | - for l in f: | ||
| 66 | - pass | ||
| 67 | - else: | ||
| 68 | - for i in range(100): | ||
| 69 | - f.seek(i*100, 0) | ||
| 70 | - except IOError: | ||
| 71 | - pass | ||
| 72 | + if next(it): | ||
| 73 | + for l in f: | ||
| 74 | + pass | ||
| 75 | + else: | ||
| 76 | + for i in xrange(100): | ||
| 77 | + f.seek(i*100, 0) | ||
| 78 | self._run_workers(iterate, 10) | ||
| 79 | |||
| 80 | + | ||
| 81 | @unittest.skipUnless(os.name == 'posix', 'test requires a posix system.') | ||
| 82 | class TestFileSignalEINTR(unittest.TestCase): | ||
| 83 | def _test_reading(self, data_to_write, read_and_verify_code, method_name, | ||
| 84 | diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst | ||
| 85 | index a6cb6c9e9b..beb09b5ae6 100644 | ||
| 86 | --- a/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst | ||
| 87 | +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst | ||
| 88 | @@ -1,4 +1 @@ | ||
| 89 | Fixed crashes when iterating over a file on multiple threads. | ||
| 90 | -seek() and next() methods of file objects now raise an exception during | ||
| 91 | -concurrent operation on the same file object. | ||
| 92 | -A lock can be used to prevent the error. | ||
| 93 | diff --git a/Objects/fileobject.c b/Objects/fileobject.c | ||
| 94 | index 8d1c5812f0..270b28264a 100644 | ||
| 95 | --- a/Objects/fileobject.c | ||
| 96 | +++ b/Objects/fileobject.c | ||
| 97 | @@ -609,7 +609,12 @@ err_iterbuffered(void) | ||
| 98 | return NULL; | ||
| 99 | } | ||
| 100 | |||
| 101 | -static void drop_readahead(PyFileObject *); | ||
| 102 | +static void | ||
| 103 | +drop_file_readahead(PyFileObject *f) | ||
| 104 | +{ | ||
| 105 | + PyMem_FREE(f->f_buf); | ||
| 106 | + f->f_buf = NULL; | ||
| 107 | +} | ||
| 108 | |||
| 109 | /* Methods */ | ||
| 110 | |||
| 111 | @@ -632,7 +637,7 @@ file_dealloc(PyFileObject *f) | ||
| 112 | Py_XDECREF(f->f_mode); | ||
| 113 | Py_XDECREF(f->f_encoding); | ||
| 114 | Py_XDECREF(f->f_errors); | ||
| 115 | - drop_readahead(f); | ||
| 116 | + drop_file_readahead(f); | ||
| 117 | Py_TYPE(f)->tp_free((PyObject *)f); | ||
| 118 | } | ||
| 119 | |||
| 120 | @@ -767,13 +772,7 @@ file_seek(PyFileObject *f, PyObject *args) | ||
| 121 | |||
| 122 | if (f->f_fp == NULL) | ||
| 123 | return err_closed(); | ||
| 124 | - if (f->unlocked_count > 0) { | ||
| 125 | - PyErr_SetString(PyExc_IOError, | ||
| 126 | - "seek() called during concurrent " | ||
| 127 | - "operation on the same file object"); | ||
| 128 | - return NULL; | ||
| 129 | - } | ||
| 130 | - drop_readahead(f); | ||
| 131 | + drop_file_readahead(f); | ||
| 132 | whence = 0; | ||
| 133 | if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &whence)) | ||
| 134 | return NULL; | ||
| 135 | @@ -2242,12 +2241,16 @@ static PyGetSetDef file_getsetlist[] = { | ||
| 136 | {0}, | ||
| 137 | }; | ||
| 138 | |||
| 139 | +typedef struct { | ||
| 140 | + char *buf, *bufptr, *bufend; | ||
| 141 | +} readaheadbuffer; | ||
| 142 | + | ||
| 143 | static void | ||
| 144 | -drop_readahead(PyFileObject *f) | ||
| 145 | +drop_readaheadbuffer(readaheadbuffer *rab) | ||
| 146 | { | ||
| 147 | - if (f->f_buf != NULL) { | ||
| 148 | - PyMem_Free(f->f_buf); | ||
| 149 | - f->f_buf = NULL; | ||
| 150 | + if (rab->buf != NULL) { | ||
| 151 | + PyMem_FREE(rab->buf); | ||
| 152 | + rab->buf = NULL; | ||
| 153 | } | ||
| 154 | } | ||
| 155 | |||
| 156 | @@ -2255,36 +2258,34 @@ drop_readahead(PyFileObject *f) | ||
| 157 | (unless at EOF) and no more than bufsize. Returns negative value on | ||
| 158 | error, will set MemoryError if bufsize bytes cannot be allocated. */ | ||
| 159 | static int | ||
| 160 | -readahead(PyFileObject *f, Py_ssize_t bufsize) | ||
| 161 | +readahead(PyFileObject *f, readaheadbuffer *rab, Py_ssize_t bufsize) | ||
| 162 | { | ||
| 163 | Py_ssize_t chunksize; | ||
| 164 | |||
| 165 | - assert(f->unlocked_count == 0); | ||
| 166 | - if (f->f_buf != NULL) { | ||
| 167 | - if( (f->f_bufend - f->f_bufptr) >= 1) | ||
| 168 | + if (rab->buf != NULL) { | ||
| 169 | + if ((rab->bufend - rab->bufptr) >= 1) | ||
| 170 | return 0; | ||
| 171 | else | ||
| 172 | - drop_readahead(f); | ||
| 173 | + drop_readaheadbuffer(rab); | ||
| 174 | } | ||
| 175 | - if ((f->f_buf = (char *)PyMem_Malloc(bufsize)) == NULL) { | ||
| 176 | + if ((rab->buf = PyMem_MALLOC(bufsize)) == NULL) { | ||
| 177 | PyErr_NoMemory(); | ||
| 178 | return -1; | ||
| 179 | } | ||
| 180 | FILE_BEGIN_ALLOW_THREADS(f) | ||
| 181 | errno = 0; | ||
| 182 | - chunksize = Py_UniversalNewlineFread( | ||
| 183 | - f->f_buf, bufsize, f->f_fp, (PyObject *)f); | ||
| 184 | + chunksize = Py_UniversalNewlineFread(rab->buf, bufsize, f->f_fp, (PyObject *)f); | ||
| 185 | FILE_END_ALLOW_THREADS(f) | ||
| 186 | if (chunksize == 0) { | ||
| 187 | if (ferror(f->f_fp)) { | ||
| 188 | PyErr_SetFromErrno(PyExc_IOError); | ||
| 189 | clearerr(f->f_fp); | ||
| 190 | - drop_readahead(f); | ||
| 191 | + drop_readaheadbuffer(rab); | ||
| 192 | return -1; | ||
| 193 | } | ||
| 194 | } | ||
| 195 | - f->f_bufptr = f->f_buf; | ||
| 196 | - f->f_bufend = f->f_buf + chunksize; | ||
| 197 | + rab->bufptr = rab->buf; | ||
| 198 | + rab->bufend = rab->buf + chunksize; | ||
| 199 | return 0; | ||
| 200 | } | ||
| 201 | |||
| 202 | @@ -2294,51 +2295,43 @@ readahead(PyFileObject *f, Py_ssize_t bufsize) | ||
| 203 | logarithmic buffer growth to about 50 even when reading a 1gb line. */ | ||
| 204 | |||
| 205 | static PyStringObject * | ||
| 206 | -readahead_get_line_skip(PyFileObject *f, Py_ssize_t skip, Py_ssize_t bufsize) | ||
| 207 | +readahead_get_line_skip(PyFileObject *f, readaheadbuffer *rab, Py_ssize_t skip, Py_ssize_t bufsize) | ||
| 208 | { | ||
| 209 | PyStringObject* s; | ||
| 210 | char *bufptr; | ||
| 211 | char *buf; | ||
| 212 | Py_ssize_t len; | ||
| 213 | |||
| 214 | - if (f->unlocked_count > 0) { | ||
| 215 | - PyErr_SetString(PyExc_IOError, | ||
| 216 | - "next() called during concurrent " | ||
| 217 | - "operation on the same file object"); | ||
| 218 | - return NULL; | ||
| 219 | - } | ||
| 220 | - if (f->f_buf == NULL) | ||
| 221 | - if (readahead(f, bufsize) < 0) | ||
| 222 | + if (rab->buf == NULL) | ||
| 223 | + if (readahead(f, rab, bufsize) < 0) | ||
| 224 | return NULL; | ||
| 225 | |||
| 226 | - len = f->f_bufend - f->f_bufptr; | ||
| 227 | + len = rab->bufend - rab->bufptr; | ||
| 228 | if (len == 0) | ||
| 229 | - return (PyStringObject *) | ||
| 230 | - PyString_FromStringAndSize(NULL, skip); | ||
| 231 | - bufptr = (char *)memchr(f->f_bufptr, '\n', len); | ||
| 232 | + return (PyStringObject *)PyString_FromStringAndSize(NULL, skip); | ||
| 233 | + bufptr = (char *)memchr(rab->bufptr, '\n', len); | ||
| 234 | if (bufptr != NULL) { | ||
| 235 | bufptr++; /* Count the '\n' */ | ||
| 236 | - len = bufptr - f->f_bufptr; | ||
| 237 | - s = (PyStringObject *) | ||
| 238 | - PyString_FromStringAndSize(NULL, skip + len); | ||
| 239 | + len = bufptr - rab->bufptr; | ||
| 240 | + s = (PyStringObject *)PyString_FromStringAndSize(NULL, skip + len); | ||
| 241 | if (s == NULL) | ||
| 242 | return NULL; | ||
| 243 | - memcpy(PyString_AS_STRING(s) + skip, f->f_bufptr, len); | ||
| 244 | - f->f_bufptr = bufptr; | ||
| 245 | - if (bufptr == f->f_bufend) | ||
| 246 | - drop_readahead(f); | ||
| 247 | + memcpy(PyString_AS_STRING(s) + skip, rab->bufptr, len); | ||
| 248 | + rab->bufptr = bufptr; | ||
| 249 | + if (bufptr == rab->bufend) | ||
| 250 | + drop_readaheadbuffer(rab); | ||
| 251 | } else { | ||
| 252 | - bufptr = f->f_bufptr; | ||
| 253 | - buf = f->f_buf; | ||
| 254 | - f->f_buf = NULL; /* Force new readahead buffer */ | ||
| 255 | + bufptr = rab->bufptr; | ||
| 256 | + buf = rab->buf; | ||
| 257 | + rab->buf = NULL; /* Force new readahead buffer */ | ||
| 258 | assert(len <= PY_SSIZE_T_MAX - skip); | ||
| 259 | - s = readahead_get_line_skip(f, skip + len, bufsize + (bufsize>>2)); | ||
| 260 | + s = readahead_get_line_skip(f, rab, skip + len, bufsize + (bufsize>>2)); | ||
| 261 | if (s == NULL) { | ||
| 262 | - PyMem_Free(buf); | ||
| 263 | + PyMem_FREE(buf); | ||
| 264 | return NULL; | ||
| 265 | } | ||
| 266 | memcpy(PyString_AS_STRING(s) + skip, bufptr, len); | ||
| 267 | - PyMem_Free(buf); | ||
| 268 | + PyMem_FREE(buf); | ||
| 269 | } | ||
| 270 | return s; | ||
| 271 | } | ||
| 272 | @@ -2356,7 +2349,30 @@ file_iternext(PyFileObject *f) | ||
| 273 | if (!f->readable) | ||
| 274 | return err_mode("reading"); | ||
| 275 | |||
| 276 | - l = readahead_get_line_skip(f, 0, READAHEAD_BUFSIZE); | ||
| 277 | + { | ||
| 278 | + /* | ||
| 279 | + Multiple threads can enter this method while the GIL is released | ||
| 280 | + during file read and wreak havoc on the file object's readahead | ||
| 281 | + buffer. To avoid dealing with cross-thread coordination issues, we | ||
| 282 | + cache the file buffer state locally and only set it back on the file | ||
| 283 | + object when we're done. | ||
| 284 | + */ | ||
| 285 | + readaheadbuffer rab = {f->f_buf, f->f_bufptr, f->f_bufend}; | ||
| 286 | + f->f_buf = NULL; | ||
| 287 | + l = readahead_get_line_skip(f, &rab, 0, READAHEAD_BUFSIZE); | ||
| 288 | + /* | ||
| 289 | + Make sure the file's internal read buffer is cleared out. This will | ||
| 290 | + only do anything if some other thread interleaved with us during | ||
| 291 | + readahead. We want to drop any changeling buffer, so we don't leak | ||
| 292 | + memory. We may lose data, but that's what you get for reading the same | ||
| 293 | + file object in multiple threads. | ||
| 294 | + */ | ||
| 295 | + drop_file_readahead(f); | ||
| 296 | + f->f_buf = rab.buf; | ||
| 297 | + f->f_bufptr = rab.bufptr; | ||
| 298 | + f->f_bufend = rab.bufend; | ||
| 299 | + } | ||
| 300 | + | ||
| 301 | if (l == NULL || PyString_GET_SIZE(l) == 0) { | ||
| 302 | Py_XDECREF(l); | ||
| 303 | return NULL; | ||
| 304 | -- | ||
| 305 | 2.13.3 | ||
| 306 | |||
diff --git a/meta/recipes-devtools/python/python/fix-gc-alignment.patch b/meta/recipes-devtools/python/python/fix-gc-alignment.patch deleted file mode 100644 index b63cd08747..0000000000 --- a/meta/recipes-devtools/python/python/fix-gc-alignment.patch +++ /dev/null | |||
| @@ -1,43 +0,0 @@ | |||
| 1 | Upstream-Status: Submitted | ||
| 2 | Signed-off-by: Ross Burton <ross.burton@intel.com> | ||
| 3 | |||
| 4 | Fix for over-aligned GC info | ||
| 5 | Patch by Florian Weimer | ||
| 6 | |||
| 7 | See: https://bugzilla.redhat.com/show_bug.cgi?id=1540316 | ||
| 8 | Upstream discussion: https://mail.python.org/pipermail/python-dev/2018-January/152000.html | ||
| 9 | |||
| 10 | diff --git a/Include/objimpl.h b/Include/objimpl.h | ||
| 11 | index 55e83eced6..aa906144dc 100644 | ||
| 12 | --- a/Include/objimpl.h | ||
| 13 | +++ b/Include/objimpl.h | ||
| 14 | @@ -248,6 +248,18 @@ PyAPI_FUNC(PyVarObject *) _PyObject_GC_Resize(PyVarObject *, Py_ssize_t); | ||
| 15 | /* for source compatibility with 2.2 */ | ||
| 16 | #define _PyObject_GC_Del PyObject_GC_Del | ||
| 17 | |||
| 18 | +/* Former over-aligned definition of PyGC_Head, used to compute the | ||
| 19 | + size of the padding for the new version below. */ | ||
| 20 | +union _gc_head; | ||
| 21 | +union _gc_head_old { | ||
| 22 | + struct { | ||
| 23 | + union _gc_head *gc_next; | ||
| 24 | + union _gc_head *gc_prev; | ||
| 25 | + Py_ssize_t gc_refs; | ||
| 26 | + } gc; | ||
| 27 | + long double dummy; | ||
| 28 | +}; | ||
| 29 | + | ||
| 30 | /* GC information is stored BEFORE the object structure. */ | ||
| 31 | typedef union _gc_head { | ||
| 32 | struct { | ||
| 33 | @@ -255,7 +267,8 @@ typedef union _gc_head { | ||
| 34 | union _gc_head *gc_prev; | ||
| 35 | Py_ssize_t gc_refs; | ||
| 36 | } gc; | ||
| 37 | - long double dummy; /* force worst-case alignment */ | ||
| 38 | + double dummy; /* force worst-case alignment */ | ||
| 39 | + char dummy_padding[sizeof(union _gc_head_old)]; | ||
| 40 | } PyGC_Head; | ||
| 41 | |||
| 42 | extern PyGC_Head *_PyGC_generation0; | ||
| 43 | \ No newline at end of file | ||
diff --git a/meta/recipes-devtools/python/python_2.7.14.bb b/meta/recipes-devtools/python/python_2.7.15.bb index b923b9237b..3044c3d363 100644 --- a/meta/recipes-devtools/python/python_2.7.14.bb +++ b/meta/recipes-devtools/python/python_2.7.15.bb | |||
| @@ -30,7 +30,6 @@ SRC_URI += "\ | |||
| 30 | file://add-CROSSPYTHONPATH-for-PYTHON_FOR_BUILD.patch \ | 30 | file://add-CROSSPYTHONPATH-for-PYTHON_FOR_BUILD.patch \ |
| 31 | file://pass-missing-libraries-to-Extension-for-mul.patch \ | 31 | file://pass-missing-libraries-to-Extension-for-mul.patch \ |
| 32 | file://support_SOURCE_DATE_EPOCH_in_py_compile_2.7.patch \ | 32 | file://support_SOURCE_DATE_EPOCH_in_py_compile_2.7.patch \ |
| 33 | file://fix-gc-alignment.patch \ | ||
| 34 | " | 33 | " |
| 35 | 34 | ||
| 36 | S = "${WORKDIR}/Python-${PV}" | 35 | S = "${WORKDIR}/Python-${PV}" |
