301 lines
10 KiB
Python
301 lines
10 KiB
Python
"""Concurrency admission tests for expensive filename walks."""
|
|
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import threading
|
|
import types
|
|
|
|
import pytest
|
|
|
|
from tools.environments.local import LocalEnvironment
|
|
from tools.file_operations import (
|
|
_ACTIVE_FILENAME_SEARCH_ROOTS,
|
|
_FILENAME_SEARCH_ADMISSION,
|
|
_normalized_filename_search_root,
|
|
SearchResult,
|
|
ShellFileOperations,
|
|
)
|
|
from tools.interrupt import set_interrupt
|
|
|
|
|
|
class RemoteEnvironment:
|
|
is_local = False
|
|
cwd = "/workspace"
|
|
|
|
def execute(self, command, **kwargs):
|
|
raise AssertionError(f"unexpected backend command: {command}")
|
|
|
|
|
|
def _operations(env, scan):
|
|
operations = ShellFileOperations(env)
|
|
operations._resolve_command = lambda command: "/usr/bin/rg" if command == "rg" else None
|
|
operations._search_files_rg = types.MethodType(scan, operations)
|
|
return operations
|
|
|
|
|
|
def test_same_backend_class_and_root_serialize_five_filename_walks():
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
counter_lock = threading.Lock()
|
|
active = 0
|
|
maximum_active = 0
|
|
completed = 0
|
|
|
|
def scan(self, pattern, path, limit, offset, order, rg_executable=None):
|
|
nonlocal active, maximum_active, completed
|
|
with counter_lock:
|
|
active += 1
|
|
maximum_active = max(maximum_active, active)
|
|
entered.set()
|
|
assert release.wait(5)
|
|
with counter_lock:
|
|
active -= 1
|
|
completed += 1
|
|
return SearchResult(files=[str(path)], total_count=1)
|
|
|
|
operations = [_operations(RemoteEnvironment(), scan) for _ in range(5)]
|
|
with ThreadPoolExecutor(max_workers=5) as pool:
|
|
futures = [
|
|
pool.submit(operation._search_files, "*.py", "/repo", 50, 0)
|
|
for operation in operations
|
|
]
|
|
assert entered.wait(5)
|
|
release.set()
|
|
results = [future.result(timeout=5) for future in futures]
|
|
|
|
assert all(result.error is None for result in results)
|
|
assert completed == 5
|
|
assert maximum_active == 1
|
|
|
|
|
|
def test_different_roots_can_enter_filename_walks_together():
|
|
both_entered = threading.Barrier(2)
|
|
|
|
def scan(self, pattern, path, limit, offset, order, rg_executable=None):
|
|
both_entered.wait(5)
|
|
return SearchResult(files=[str(path)], total_count=1)
|
|
|
|
first = _operations(RemoteEnvironment(), scan)
|
|
second = _operations(RemoteEnvironment(), scan)
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
futures = [
|
|
pool.submit(first._search_files, "*.py", "/one", 50, 0),
|
|
pool.submit(second._search_files, "*.py", "/two", 50, 0),
|
|
]
|
|
assert [future.result(timeout=5).error for future in futures] == [None, None]
|
|
|
|
|
|
def test_different_backend_classes_can_walk_the_same_root_together():
|
|
class OtherRemoteEnvironment(RemoteEnvironment):
|
|
pass
|
|
|
|
both_entered = threading.Barrier(2)
|
|
|
|
def scan(self, pattern, path, limit, offset, order, rg_executable=None):
|
|
both_entered.wait(5)
|
|
return SearchResult(files=[str(path)], total_count=1)
|
|
|
|
first = _operations(RemoteEnvironment(), scan)
|
|
second = _operations(OtherRemoteEnvironment(), scan)
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
futures = [
|
|
pool.submit(first._search_files, "*.py", "/same", 50, 0),
|
|
pool.submit(second._search_files, "*.py", "/same", 50, 0),
|
|
]
|
|
assert [future.result(timeout=5).error for future in futures] == [None, None]
|
|
|
|
|
|
def test_overlapping_multi_root_sets_are_claimed_atomically(monkeypatch):
|
|
first_entered = threading.Event()
|
|
release_first = threading.Event()
|
|
second_waiting = threading.Event()
|
|
lock = threading.Lock()
|
|
active = 0
|
|
maximum_active = 0
|
|
|
|
def scan(self, pattern, path, limit, offset, order, rg_executable=None):
|
|
nonlocal active, maximum_active
|
|
with lock:
|
|
active += 1
|
|
maximum_active = max(maximum_active, active)
|
|
if path == ["/a", "/b"]:
|
|
first_entered.set()
|
|
if path == ["/a", "/b"]:
|
|
assert release_first.wait(5)
|
|
with lock:
|
|
active -= 1
|
|
return SearchResult(files=[str(path)], total_count=1)
|
|
|
|
first = _operations(RemoteEnvironment(), scan)
|
|
second = _operations(RemoteEnvironment(), scan)
|
|
original_wait = _FILENAME_SEARCH_ADMISSION.wait
|
|
|
|
def observed_wait(timeout=None):
|
|
second_waiting.set()
|
|
return original_wait(timeout)
|
|
|
|
monkeypatch.setattr(_FILENAME_SEARCH_ADMISSION, "wait", observed_wait)
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
first_future = pool.submit(first._search_files, "*.py", ["/a", "/b"], 50, 0)
|
|
assert first_entered.wait(5)
|
|
second_future = pool.submit(second._search_files, "*.py", ["/b", "/c"], 50, 0)
|
|
assert second_waiting.wait(5)
|
|
release_first.set()
|
|
assert first_future.result(timeout=5).error is None
|
|
assert second_future.result(timeout=5).error is None
|
|
|
|
assert maximum_active == 1
|
|
|
|
|
|
def test_interrupted_waiter_returns_without_dispatch_or_late_dispatch(monkeypatch):
|
|
holder_entered = threading.Event()
|
|
release_holder = threading.Event()
|
|
waiter_waiting = threading.Event()
|
|
waiter_tid = []
|
|
dispatches = []
|
|
|
|
def scan(self, pattern, path, limit, offset, order, rg_executable=None):
|
|
dispatches.append(threading.get_ident())
|
|
holder_entered.set()
|
|
assert release_holder.wait(5)
|
|
return SearchResult(files=[str(path)], total_count=1)
|
|
|
|
holder = _operations(RemoteEnvironment(), scan)
|
|
waiter = _operations(RemoteEnvironment(), scan)
|
|
|
|
original_wait = _FILENAME_SEARCH_ADMISSION.wait
|
|
|
|
def observed_wait(timeout=None):
|
|
waiter_waiting.set()
|
|
return original_wait(timeout)
|
|
|
|
monkeypatch.setattr(_FILENAME_SEARCH_ADMISSION, "wait", observed_wait)
|
|
|
|
def run_waiter():
|
|
waiter_tid.append(threading.get_ident())
|
|
return waiter._search_files("*.py", "/repo", 50, 0)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
holder_future = pool.submit(holder._search_files, "*.py", "/repo", 50, 0)
|
|
assert holder_entered.wait(5)
|
|
waiter_future = pool.submit(run_waiter)
|
|
assert waiter_waiting.wait(5)
|
|
set_interrupt(True, waiter_tid[0])
|
|
try:
|
|
interrupted = waiter_future.result(timeout=5)
|
|
assert "interrupted" in (interrupted.error or "").lower()
|
|
assert len(dispatches) == 1
|
|
release_holder.set()
|
|
assert holder_future.result(timeout=5).error is None
|
|
assert len(dispatches) == 1
|
|
finally:
|
|
set_interrupt(False, waiter_tid[0])
|
|
release_holder.set()
|
|
|
|
|
|
def test_interrupt_published_after_final_sample_prevents_filename_dispatch(monkeypatch):
|
|
sampled_clear = threading.Event()
|
|
resume_acquire = threading.Event()
|
|
worker_tid = []
|
|
dispatches = []
|
|
|
|
def scan(self, pattern, path, limit, offset, order, rg_executable=None):
|
|
dispatches.append(threading.get_ident())
|
|
return SearchResult(files=[str(path)], total_count=1)
|
|
|
|
operations = _operations(RemoteEnvironment(), scan)
|
|
original_is_interrupted = __import__(
|
|
"tools.interrupt", fromlist=["is_interrupted"]
|
|
).is_interrupted
|
|
|
|
def pause_after_clear_sample():
|
|
interrupted = original_is_interrupted()
|
|
if not interrupted and threading.get_ident() == worker_tid[0]:
|
|
sampled_clear.set()
|
|
assert resume_acquire.wait(5)
|
|
return interrupted
|
|
|
|
monkeypatch.setattr(
|
|
"tools.file_operations.tool_interrupt.is_interrupted",
|
|
pause_after_clear_sample,
|
|
)
|
|
|
|
def run_search():
|
|
worker_tid.append(threading.get_ident())
|
|
return operations._search_files("*.py", "/repo", 50, 0)
|
|
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
future = pool.submit(run_search)
|
|
assert sampled_clear.wait(5)
|
|
set_interrupt(True, worker_tid[0])
|
|
resume_acquire.set()
|
|
try:
|
|
result = future.result(timeout=5)
|
|
finally:
|
|
set_interrupt(False, worker_tid[0])
|
|
resume_acquire.set()
|
|
|
|
assert "interrupted" in (result.error or "").lower()
|
|
assert dispatches == []
|
|
assert _ACTIVE_FILENAME_SEARCH_ROOTS == set()
|
|
|
|
|
|
def test_empty_filename_roots_are_rejected_before_engine_resolution():
|
|
def scan(self, pattern, path, limit, offset, order, rg_executable=None):
|
|
raise AssertionError("filename engine dispatched")
|
|
|
|
operations = _operations(RemoteEnvironment(), scan)
|
|
operations._resolve_command = lambda command: (_ for _ in ()).throw(
|
|
AssertionError(f"engine resolution attempted: {command}")
|
|
)
|
|
|
|
result = operations._search_files("*.py", [], 50, 0)
|
|
|
|
assert "at least one search root" in (result.error or "").lower()
|
|
assert _ACTIVE_FILENAME_SEARCH_ROOTS == set()
|
|
|
|
|
|
@pytest.mark.parametrize("raised", [Exception, KeyboardInterrupt, SystemExit, BaseException])
|
|
def test_admission_releases_after_every_base_exception_path(raised):
|
|
attempts = 0
|
|
|
|
def scan(self, pattern, path, limit, offset, order, rg_executable=None):
|
|
nonlocal attempts
|
|
attempts += 1
|
|
if attempts == 1:
|
|
raise raised("engine failed")
|
|
return SearchResult(files=[str(path)], total_count=1)
|
|
|
|
operations = _operations(RemoteEnvironment(), scan)
|
|
with pytest.raises(raised, match="engine failed"):
|
|
operations._search_files("*.py", "/repo", 50, 0)
|
|
|
|
result = operations._search_files("*.py", "/repo", 50, 0)
|
|
assert result.error is None
|
|
assert attempts == 2
|
|
assert _ACTIVE_FILENAME_SEARCH_ROOTS == set()
|
|
|
|
|
|
def test_remote_roots_are_normalized_lexically_against_backend_cwd(monkeypatch):
|
|
env = RemoteEnvironment()
|
|
monkeypatch.setattr(
|
|
"tools.file_operations.os.path.abspath",
|
|
lambda path: (_ for _ in ()).throw(AssertionError("controller resolution used")),
|
|
)
|
|
|
|
relative = _normalized_filename_search_root(env, "repo/../repo", "/controller")
|
|
absolute = _normalized_filename_search_root(env, "/workspace/repo", "/controller")
|
|
|
|
assert relative == "/workspace/repo"
|
|
assert absolute == relative
|
|
|
|
|
|
@pytest.mark.windows_only
|
|
def test_windows_local_root_spellings_share_one_normalized_key():
|
|
env = LocalEnvironment.__new__(LocalEnvironment)
|
|
env.cwd = "C:/Repo"
|
|
|
|
native = _normalized_filename_search_root(env, r"C:\Repo\src\..", "C:/ignored")
|
|
msys = _normalized_filename_search_root(env, "/c/Repo", "C:/ignored")
|
|
|
|
assert native == msys
|