Skip to content

Commit c7ae9eb

Browse files
tkrenpitrou
authored andcommitted
shared_memory: avoid quadratic item access performance of ShareableList
Avoid linear runtime of ShareableList.__getitem__ and ShareableList.__setitem__ by storing running allocated bytes in ShareableList._allocated_bytes instead of the number of bytes for a particular stored item.
1 parent 1ac6e37 commit c7ae9eb

1 file changed

Lines changed: 21 additions & 9 deletions

File tree

Lib/multiprocessing/shared_memory.py

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -294,9 +294,13 @@ def __init__(self, sequence=None, *, name=None):
294294
]
295295
self._list_len = len(_formats)
296296
assert sum(len(fmt) <= 8 for fmt in _formats) == self._list_len
297+
sum_allocated_bytes = 0
297298
self._allocated_bytes = tuple(
298-
self._alignment if fmt[-1] != "s" else int(fmt[:-1])
299-
for fmt in _formats
299+
(sum_allocated_bytes := sum_allocated_bytes + (
300+
self._alignment if fmt[-1] != "s" else int(fmt[:-1])
301+
)
302+
)
303+
for fmt in _formats
300304
)
301305
_recreation_codes = [
302306
self._extract_recreation_code(item) for item in sequence
@@ -409,8 +413,11 @@ def _set_packing_format_and_transform(self, position, fmt_as_str, value):
409413

410414
def __getitem__(self, position):
411415
try:
412-
offset = self._offset_data_start \
413-
+ sum(self._allocated_bytes[:position])
416+
if position == 0:
417+
allocated_predecessor = 0
418+
else:
419+
allocated_predecessor = self._allocated_bytes[position - 1]
420+
offset = self._offset_data_start + allocated_predecessor
414421
(v,) = struct.unpack_from(
415422
self._get_packing_format(position),
416423
self.shm.buf,
@@ -426,22 +433,27 @@ def __getitem__(self, position):
426433

427434
def __setitem__(self, position, value):
428435
try:
429-
offset = self._offset_data_start \
430-
+ sum(self._allocated_bytes[:position])
436+
if position == 0:
437+
allocated_predecessor = 0
438+
else:
439+
allocated_predecessor = self._allocated_bytes[position - 1]
440+
offset = self._offset_data_start + allocated_predecessor
431441
current_format = self._get_packing_format(position)
432442
except IndexError:
433443
raise IndexError("assignment index out of range")
434444

435445
if not isinstance(value, (str, bytes)):
436446
new_format = self._types_mapping[type(value)]
437447
else:
438-
if len(value) > self._allocated_bytes[position]:
448+
allocated_position = self._allocated_bytes[position] - allocated_predecessor
449+
450+
if len(value) > allocated_position:
439451
raise ValueError("exceeds available storage for existing str")
440452
if current_format[-1] == "s":
441453
new_format = current_format
442454
else:
443455
new_format = self._types_mapping[str] % (
444-
self._allocated_bytes[position],
456+
allocated_position,
445457
)
446458

447459
self._set_packing_format_and_transform(
@@ -489,7 +501,7 @@ def _offset_data_start(self):
489501

490502
@property
491503
def _offset_packing_formats(self):
492-
return self._offset_data_start + sum(self._allocated_bytes)
504+
return self._offset_data_start + self._allocated_bytes[-1]
493505

494506
@property
495507
def _offset_back_transform_codes(self):

0 commit comments

Comments
 (0)