mirror of
https://github.com/thegeeklab/ansible-later.git
synced 2024-11-15 01:30:40 +00:00
227 lines
11 KiB
Python
227 lines
11 KiB
Python
|
from __future__ import print_function
|
||
|
|
|
||
|
|
from .lib import TestBase, FileCreator
|
||
|
|
|
||
|
|
from smmap.mman import (
|
||
|
|
WindowCursor,
|
||
|
|
SlidingWindowMapManager,
|
||
|
|
StaticWindowMapManager
|
||
|
|
)
|
||
|
|
from smmap.util import align_to_mmap
|
||
|
|
|
||
|
|
from random import randint
|
||
|
|
from time import time
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
from copy import copy
|
||
|
|
|
||
|
|
|
||
|
|
class TestMMan(TestBase):
|
||
|
|
|
||
|
|
def test_cursor(self):
|
||
|
|
with FileCreator(self.k_window_test_size, "cursor_test") as fc:
|
||
|
|
man = SlidingWindowMapManager()
|
||
|
|
ci = WindowCursor(man) # invalid cursor
|
||
|
|
assert not ci.is_valid()
|
||
|
|
assert not ci.is_associated()
|
||
|
|
assert ci.size() == 0 # this is cached, so we can query it in invalid state
|
||
|
|
|
||
|
|
cv = man.make_cursor(fc.path)
|
||
|
|
assert not cv.is_valid() # no region mapped yet
|
||
|
|
assert cv.is_associated() # but it know where to map it from
|
||
|
|
assert cv.file_size() == fc.size
|
||
|
|
assert cv.path() == fc.path
|
||
|
|
|
||
|
|
# copy module
|
||
|
|
cio = copy(cv)
|
||
|
|
assert not cio.is_valid() and cio.is_associated()
|
||
|
|
|
||
|
|
# assign method
|
||
|
|
assert not ci.is_associated()
|
||
|
|
ci.assign(cv)
|
||
|
|
assert not ci.is_valid() and ci.is_associated()
|
||
|
|
|
||
|
|
# unuse non-existing region is fine
|
||
|
|
cv.unuse_region()
|
||
|
|
cv.unuse_region()
|
||
|
|
|
||
|
|
# destruction is fine (even multiple times)
|
||
|
|
cv._destroy()
|
||
|
|
WindowCursor(man)._destroy()
|
||
|
|
|
||
|
|
def test_memory_manager(self):
|
||
|
|
slide_man = SlidingWindowMapManager()
|
||
|
|
static_man = StaticWindowMapManager()
|
||
|
|
|
||
|
|
for man in (static_man, slide_man):
|
||
|
|
assert man.num_file_handles() == 0
|
||
|
|
assert man.num_open_files() == 0
|
||
|
|
winsize_cmp_val = 0
|
||
|
|
if isinstance(man, StaticWindowMapManager):
|
||
|
|
winsize_cmp_val = -1
|
||
|
|
# END handle window size
|
||
|
|
assert man.window_size() > winsize_cmp_val
|
||
|
|
assert man.mapped_memory_size() == 0
|
||
|
|
assert man.max_mapped_memory_size() > 0
|
||
|
|
|
||
|
|
# collection doesn't raise in 'any' mode
|
||
|
|
man._collect_lru_region(0)
|
||
|
|
# doesn't raise if we are within the limit
|
||
|
|
man._collect_lru_region(10)
|
||
|
|
|
||
|
|
# doesn't fail if we over-allocate
|
||
|
|
assert man._collect_lru_region(sys.maxsize) == 0
|
||
|
|
|
||
|
|
# use a region, verify most basic functionality
|
||
|
|
with FileCreator(self.k_window_test_size, "manager_test") as fc:
|
||
|
|
fd = os.open(fc.path, os.O_RDONLY)
|
||
|
|
try:
|
||
|
|
for item in (fc.path, fd):
|
||
|
|
c = man.make_cursor(item)
|
||
|
|
assert c.path_or_fd() is item
|
||
|
|
assert c.use_region(10, 10).is_valid()
|
||
|
|
assert c.ofs_begin() == 10
|
||
|
|
assert c.size() == 10
|
||
|
|
with open(fc.path, 'rb') as fp:
|
||
|
|
assert c.buffer()[:] == fp.read(20)[10:]
|
||
|
|
|
||
|
|
if isinstance(item, int):
|
||
|
|
self.assertRaises(ValueError, c.path)
|
||
|
|
else:
|
||
|
|
self.assertRaises(ValueError, c.fd)
|
||
|
|
# END handle value error
|
||
|
|
# END for each input
|
||
|
|
finally:
|
||
|
|
os.close(fd)
|
||
|
|
# END for each manasger type
|
||
|
|
|
||
|
|
def test_memman_operation(self):
|
||
|
|
# test more access, force it to actually unmap regions
|
||
|
|
with FileCreator(self.k_window_test_size, "manager_operation_test") as fc:
|
||
|
|
with open(fc.path, 'rb') as fp:
|
||
|
|
data = fp.read()
|
||
|
|
fd = os.open(fc.path, os.O_RDONLY)
|
||
|
|
try:
|
||
|
|
max_num_handles = 15
|
||
|
|
# small_size =
|
||
|
|
for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)),
|
||
|
|
(SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),):
|
||
|
|
for item in (fc.path, fd):
|
||
|
|
assert len(data) == fc.size
|
||
|
|
|
||
|
|
# small windows, a reasonable max memory. Not too many regions at once
|
||
|
|
man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2])
|
||
|
|
c = man.make_cursor(item)
|
||
|
|
|
||
|
|
# still empty (more about that is tested in test_memory_manager()
|
||
|
|
assert man.num_open_files() == 0
|
||
|
|
assert man.mapped_memory_size() == 0
|
||
|
|
|
||
|
|
base_offset = 5000
|
||
|
|
# window size is 0 for static managers, hence size will be 0. We take that into consideration
|
||
|
|
size = man.window_size() // 2
|
||
|
|
assert c.use_region(base_offset, size).is_valid()
|
||
|
|
rr = c.region()
|
||
|
|
assert rr.client_count() == 2 # the manager and the cursor and us
|
||
|
|
|
||
|
|
assert man.num_open_files() == 1
|
||
|
|
assert man.num_file_handles() == 1
|
||
|
|
assert man.mapped_memory_size() == rr.size()
|
||
|
|
|
||
|
|
# assert c.size() == size # the cursor may overallocate in its static version
|
||
|
|
assert c.ofs_begin() == base_offset
|
||
|
|
assert rr.ofs_begin() == 0 # it was aligned and expanded
|
||
|
|
if man.window_size():
|
||
|
|
# but isn't larger than the max window (aligned)
|
||
|
|
assert rr.size() == align_to_mmap(man.window_size(), True)
|
||
|
|
else:
|
||
|
|
assert rr.size() == fc.size
|
||
|
|
# END ignore static managers which dont use windows and are aligned to file boundaries
|
||
|
|
|
||
|
|
assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())]
|
||
|
|
|
||
|
|
# obtain second window, which spans the first part of the file - it is a still the same window
|
||
|
|
nsize = (size or fc.size) - 10
|
||
|
|
assert c.use_region(0, nsize).is_valid()
|
||
|
|
assert c.region() == rr
|
||
|
|
assert man.num_file_handles() == 1
|
||
|
|
assert c.size() == nsize
|
||
|
|
assert c.ofs_begin() == 0
|
||
|
|
assert c.buffer()[:] == data[:nsize]
|
||
|
|
|
||
|
|
# map some part at the end, our requested size cannot be kept
|
||
|
|
overshoot = 4000
|
||
|
|
base_offset = fc.size - (size or c.size()) + overshoot
|
||
|
|
assert c.use_region(base_offset, size).is_valid()
|
||
|
|
if man.window_size():
|
||
|
|
assert man.num_file_handles() == 2
|
||
|
|
assert c.size() < size
|
||
|
|
assert c.region() is not rr # old region is still available, but has not curser ref anymore
|
||
|
|
assert rr.client_count() == 1 # only held by manager
|
||
|
|
else:
|
||
|
|
assert c.size() < fc.size
|
||
|
|
# END ignore static managers which only have one handle per file
|
||
|
|
rr = c.region()
|
||
|
|
assert rr.client_count() == 2 # manager + cursor
|
||
|
|
assert rr.ofs_begin() < c.ofs_begin() # it should have extended itself to the left
|
||
|
|
assert rr.ofs_end() <= fc.size # it cannot be larger than the file
|
||
|
|
assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())]
|
||
|
|
|
||
|
|
# unising a region makes the cursor invalid
|
||
|
|
c.unuse_region()
|
||
|
|
assert not c.is_valid()
|
||
|
|
if man.window_size():
|
||
|
|
# but doesn't change anything regarding the handle count - we cache it and only
|
||
|
|
# remove mapped regions if we have to
|
||
|
|
assert man.num_file_handles() == 2
|
||
|
|
# END ignore this for static managers
|
||
|
|
|
||
|
|
# iterate through the windows, verify data contents
|
||
|
|
# this will trigger map collection after a while
|
||
|
|
max_random_accesses = 5000
|
||
|
|
num_random_accesses = max_random_accesses
|
||
|
|
memory_read = 0
|
||
|
|
st = time()
|
||
|
|
|
||
|
|
# cache everything to get some more performance
|
||
|
|
includes_ofs = c.includes_ofs
|
||
|
|
max_mapped_memory_size = man.max_mapped_memory_size()
|
||
|
|
max_file_handles = man.max_file_handles()
|
||
|
|
mapped_memory_size = man.mapped_memory_size
|
||
|
|
num_file_handles = man.num_file_handles
|
||
|
|
while num_random_accesses:
|
||
|
|
num_random_accesses -= 1
|
||
|
|
base_offset = randint(0, fc.size - 1)
|
||
|
|
|
||
|
|
# precondition
|
||
|
|
if man.window_size():
|
||
|
|
assert max_mapped_memory_size >= mapped_memory_size()
|
||
|
|
# END statics will overshoot, which is fine
|
||
|
|
assert max_file_handles >= num_file_handles()
|
||
|
|
assert c.use_region(base_offset, (size or c.size())).is_valid()
|
||
|
|
csize = c.size()
|
||
|
|
assert c.buffer()[:] == data[base_offset:base_offset + csize]
|
||
|
|
memory_read += csize
|
||
|
|
|
||
|
|
assert includes_ofs(base_offset)
|
||
|
|
assert includes_ofs(base_offset + csize - 1)
|
||
|
|
assert not includes_ofs(base_offset + csize)
|
||
|
|
# END while we should do an access
|
||
|
|
elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows
|
||
|
|
mb = float(1000 * 1000)
|
||
|
|
print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n"
|
||
|
|
% (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed),
|
||
|
|
file=sys.stderr)
|
||
|
|
|
||
|
|
# an offset as large as the size doesn't work !
|
||
|
|
assert not c.use_region(fc.size, size).is_valid()
|
||
|
|
|
||
|
|
# collection - it should be able to collect all
|
||
|
|
assert man.num_file_handles()
|
||
|
|
assert man.collect()
|
||
|
|
assert man.num_file_handles() == 0
|
||
|
|
# END for each item
|
||
|
|
# END for each manager type
|
||
|
|
finally:
|
||
|
|
os.close(fd)
|