import asyncio
from asyncio.subprocess import SubprocessStreamProtocol
import contextlib
import functools
import os
import pathlib
import psutil
import signal
import subprocess
from typing import Dict, List, Optional, Tuple
from ceph_devstack import logger, VERBOSE
_TERMINATE_TIMEOUT = 3.0
_KILL_TIMEOUT = 1.0
class Subprocess(asyncio.subprocess.Process):
async def _close_transport(self) -> None:
transport = getattr(self, "_transport", None)
if transport is not None and not transport.is_closing():
transport.close()
async def _wait_for_exit(self, timeout: float) -> None:
with contextlib.suppress(asyncio.TimeoutError, asyncio.CancelledError):
await asyncio.wait_for(asyncio.shield(super().wait()), timeout=timeout)
async def _terminate(self) -> None:
if self.returncode is not None:
await self._close_transport()
return
self.signal_children(signal.SIGTERM, recursive=True)
with contextlib.suppress(ProcessLookupError):
self.kill()
await self._wait_for_exit(_TERMINATE_TIMEOUT)
if self.returncode is None:
self.signal_children(signal.SIGKILL, recursive=True)
with contextlib.suppress(ProcessLookupError):
self.kill()
await self._wait_for_exit(_KILL_TIMEOUT)
await self._close_transport()
async def wait(self) -> int:
try:
return await super().wait()
except asyncio.CancelledError:
await self._terminate()
raise
async def communicate(self, input=None):
try:
return await super().communicate(input)
except asyncio.CancelledError:
await self._terminate()
raise
def child_pids(self, recursive=True):
if self.pid is None:
return []
return [
child.pid
for child in psutil.Process(self.pid).children(recursive=recursive)
]
def signal_children(self, signal: signal.Signals, recursive=True):
for pid in self.child_pids(recursive=recursive):
with contextlib.suppress(ProcessLookupError):
os.kill(pid, signal)
with contextlib.suppress(ProcessLookupError):
os.killpg(pid, signal)
async def collect_output(self) -> Tuple[str, str]:
stdout = b""
stderr = b""
if self.stdout is not None:
stdout = await self.stdout.read()
if self.stderr is not None:
stderr = await self.stderr.read()
return stdout.decode(), stderr.decode()
async def log_failure(self, cmd: List[str]) -> Tuple[str, str]:
stdout, stderr = await self.collect_output()
returncode = self.returncode if self.returncode is not None else -1
logger.error(f"Command failed ({returncode}): {' '.join(cmd)}")
for line in stderr.rstrip("\n").splitlines():
logger.error(line)
for line in stdout.rstrip("\n").splitlines():
logger.error(line)
return stdout, stderr
class Command:
def __init__(
self,
args: List[str],
cwd: Optional[pathlib.Path] = None,
env: Optional[Dict] = None,
stream_output: bool = False,
):
self.args = args
self.env = os.environ | (env or {})
self.kwargs: Dict = {
"stdout": asyncio.subprocess.PIPE,
"stderr": asyncio.subprocess.PIPE,
}
if cwd:
self.kwargs.update(cwd=cwd)
self.stream_output = stream_output
def _make_log_msg(self) -> str:
msg = "> " + " ".join(self.args)
if (cwd := str(self.kwargs.get("cwd", "."))) != ".":
msg = f"{msg} cwd='{cwd}'"
return msg
def run(self) -> subprocess.Popen:
logger.log(VERBOSE, self._make_log_msg())
proc = subprocess.Popen(
args=self.args,
env=self.env,
**self.kwargs,
)
proc.wait()
return proc
async def arun(self) -> Subprocess:
logger.log(VERBOSE, self._make_log_msg())
loop = asyncio.get_running_loop()
kwargs = dict(self.kwargs)
if self.stream_output:
# Inherit stdout/stderr so long-running commands (e.g. dnf builddep)
# are not blocked when their output exceeds the StreamReader limit.
kwargs["stdout"] = None
kwargs["stderr"] = None
protocol_factory = functools.partial(
SubprocessStreamProtocol,
limit=2**16,
loop=loop,
)
transport, protocol = await loop.subprocess_exec(
protocol_factory,
*self.args,
env=self.env,
start_new_session=True,
**kwargs,
)
return Subprocess(transport, protocol, loop)
def __str__(self):
return " ".join(self.args)