zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change master/child to controller/minion in pty docstrings

Browse Source 
PTY support used the concept of 'master' and 'child' processes. 'master'
has been renamed to 'controller' and 'child' to 'minion'.
This commit is contained in:
Todd Gamblin 2020-06-30 23:54:29 -07:00
parent f0275d7e1b
commit 4ea76dc95c
2 changed files with 97 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
lib/spack/llnl/util/tty/pty.py
View File

@ -31,17 +31,17 @@
class ProcessController(object): class ProcessController(object):
"""Wrapper around some fundamental process control operations. """Wrapper around some fundamental process control operations.
This allows one process to drive another similar to the way a shell This allows one process (the controller) to drive another (the
would, by sending signals and I/O. minion) similar to the way a shell would, by sending signals and I/O.
""" """
def __init__(self, pid, master_fd, def __init__(self, pid, controller_fd,
timeout=1, sleep_time=1e-1, debug=False): timeout=1, sleep_time=1e-1, debug=False):
"""Create a controller to manipulate the process with id ``pid`` """Create a controller to manipulate the process with id ``pid``
Args: Args:
pid (int): id of process to control pid (int): id of process to control
master_fd (int): master file descriptor attached to pid's stdin controller_fd (int): controller fd attached to pid's stdin
timeout (int): time in seconds for wait operations to time out timeout (int): time in seconds for wait operations to time out
(default 1 second) (default 1 second)
sleep_time (int): time to sleep after signals, to control the sleep_time (int): time to sleep after signals, to control the
@ -58,7 +58,7 @@ def __init__(self, pid, master_fd,
""" """
self.pid = pid self.pid = pid
self.pgid = os.getpgid(pid) self.pgid = os.getpgid(pid)
self.master_fd = master_fd self.controller_fd = controller_fd
self.timeout = timeout self.timeout = timeout
self.sleep_time = sleep_time self.sleep_time = sleep_time
self.debug = debug self.debug = debug
@ -67,8 +67,8 @@ def __init__(self, pid, master_fd,
self.ps = which("ps", required=True) self.ps = which("ps", required=True)
def get_canon_echo_attrs(self): def get_canon_echo_attrs(self):
"""Get echo and canon attributes of the terminal of master_fd.""" """Get echo and canon attributes of the terminal of controller_fd."""
cfg = termios.tcgetattr(self.master_fd) cfg = termios.tcgetattr(self.controller_fd)
return ( return (
bool(cfg[3] & termios.ICANON), bool(cfg[3] & termios.ICANON),
bool(cfg[3] & termios.ECHO), bool(cfg[3] & termios.ECHO),
@ -82,7 +82,7 @@ def horizontal_line(self, name):
) )
def status(self): def status(self):
"""Print debug message with status info for the child.""" """Print debug message with status info for the minion."""
if self.debug: if self.debug:
canon, echo = self.get_canon_echo_attrs() canon, echo = self.get_canon_echo_attrs()
sys.stderr.write("canon: %s, echo: %s\n" % ( sys.stderr.write("canon: %s, echo: %s\n" % (
@ -94,12 +94,12 @@ def status(self):
sys.stderr.write("\n") sys.stderr.write("\n")
def input_on(self): def input_on(self):
"""True if keyboard input is enabled on the master_fd pty.""" """True if keyboard input is enabled on the controller_fd pty."""
return self.get_canon_echo_attrs() == (False, False) return self.get_canon_echo_attrs() == (False, False)
def background(self): def background(self):
"""True if pgid is in a background pgroup of master_fd's terminal.""" """True if pgid is in a background pgroup of controller_fd's tty."""
return self.pgid != os.tcgetpgrp(self.master_fd) return self.pgid != os.tcgetpgrp(self.controller_fd)
def tstp(self): def tstp(self):
"""Send SIGTSTP to the controlled process.""" """Send SIGTSTP to the controlled process."""
@ -115,18 +115,18 @@ def cont(self):
def fg(self): def fg(self):
self.horizontal_line("fg") self.horizontal_line("fg")
with log.ignore_signal(signal.SIGTTOU): with log.ignore_signal(signal.SIGTTOU):
os.tcsetpgrp(self.master_fd, os.getpgid(self.pid)) os.tcsetpgrp(self.controller_fd, os.getpgid(self.pid))
time.sleep(self.sleep_time) time.sleep(self.sleep_time)
def bg(self): def bg(self):
self.horizontal_line("bg") self.horizontal_line("bg")
with log.ignore_signal(signal.SIGTTOU): with log.ignore_signal(signal.SIGTTOU):
os.tcsetpgrp(self.master_fd, os.getpgrp()) os.tcsetpgrp(self.controller_fd, os.getpgrp())
time.sleep(self.sleep_time) time.sleep(self.sleep_time)
def write(self, byte_string): def write(self, byte_string):
self.horizontal_line("write '%s'" % byte_string.decode("utf-8")) self.horizontal_line("write '%s'" % byte_string.decode("utf-8"))
os.write(self.master_fd, byte_string) os.write(self.controller_fd, byte_string)
def wait(self, condition): def wait(self, condition):
start = time.time() start = time.time()
@ -156,50 +156,51 @@ def wait_running(self):
class PseudoShell(object): class PseudoShell(object):
"""Sets up master and child processes with a PTY. """Sets up controller and minion processes with a PTY.
You can create a ``PseudoShell`` if you want to test how some You can create a ``PseudoShell`` if you want to test how some
function responds to terminal input. This is a pseudo-shell from a function responds to terminal input. This is a pseudo-shell from a
job control perspective; ``master_function`` and ``child_function`` job control perspective; ``controller_function`` and ``minion_function``
are set up with a pseudoterminal (pty) so that the master can drive are set up with a pseudoterminal (pty) so that the controller can drive
the child through process control signals and I/O. the minion through process control signals and I/O.
The two functions should have signatures like this:: The two functions should have signatures like this::
def master_function(proc, ctl, **kwargs) def controller_function(proc, ctl, **kwargs)
def child_function(**kwargs) def minion_function(**kwargs)
``master_function`` is spawned in its own process and passed three ``controller_function`` is spawned in its own process and passed three
arguments: arguments:
proc proc
the ``multiprocessing.Process`` object representing the child the ``multiprocessing.Process`` object representing the minion
ctl ctl
a ``ProcessController`` object tied to the child a ``ProcessController`` object tied to the minion
kwargs kwargs
keyword arguments passed from ``PseudoShell.start()``. keyword arguments passed from ``PseudoShell.start()``.
``child_function`` is only passed ``kwargs`` delegated from ``minion_function`` is only passed ``kwargs`` delegated from
``PseudoShell.start()``. ``PseudoShell.start()``.
The ``ctl.master_fd`` will have its ``master_fd`` connected to The ``ctl.controller_fd`` will have its ``controller_fd`` connected to
``sys.stdin`` in the child process. Both processes will share the ``sys.stdin`` in the minion process. Both processes will share the
same ``sys.stdout`` and ``sys.stderr`` as the process instantiating same ``sys.stdout`` and ``sys.stderr`` as the process instantiating
``PseudoShell``. ``PseudoShell``.
Here are the relationships between processes created:: Here are the relationships between processes created::
._________________________________________________________. ._________________________________________________________.
| Child Process | pid 2 | Minion Process | pid 2
| - runs child_function | pgroup 2 | - runs minion_function | pgroup 2
|_________________________________________________________| session 1 |_________________________________________________________| session 1
^ ^
| create process with master_fd connected to stdin | create process with controller_fd connected to stdin
| stdout, stderr are the same as caller | stdout, stderr are the same as caller
._________________________________________________________. ._________________________________________________________.
| Master Process | pid 1 | Controller Process | pid 1
| - runs master_function | pgroup 1 | - runs controller_function | pgroup 1
| - uses ProcessController and master_fd to control child | session 1 | - uses ProcessController and controller_fd to | session 1
| control minion |
|_________________________________________________________| |_________________________________________________________|
^ ^
| create process | create process
@ -207,51 +208,51 @@ def child_function(**kwargs)
._________________________________________________________. ._________________________________________________________.
| Caller | pid 0 | Caller | pid 0
| - Constructs, starts, joins PseudoShell | pgroup 0 | - Constructs, starts, joins PseudoShell | pgroup 0
| - provides master_function, child_function | session 0 | - provides controller_function, minion_function | session 0
|_________________________________________________________| |_________________________________________________________|
""" """
def __init__(self, master_function, child_function): def __init__(self, controller_function, minion_function):
self.proc = None self.proc = None
self.master_function = master_function self.controller_function = controller_function
self.child_function = child_function self.minion_function = minion_function
# these can be optionally set to change defaults # these can be optionally set to change defaults
self.controller_timeout = 1 self.controller_timeout = 1
self.sleep_time = 0 self.sleep_time = 0
def start(self, **kwargs): def start(self, **kwargs):
"""Start the master and child processes. """Start the controller and minion processes.
Arguments: Arguments:
kwargs (dict): arbitrary keyword arguments that will be kwargs (dict): arbitrary keyword arguments that will be
passed to master and child functions passed to controller and minion functions
The master process will create the child, then call The controller process will create the minion, then call
``master_function``. The child process will call ``controller_function``. The minion process will call
``child_function``. ``minion_function``.
""" """
self.proc = multiprocessing.Process( self.proc = multiprocessing.Process(
target=PseudoShell._set_up_and_run_master_function, target=PseudoShell._set_up_and_run_controller_function,
args=(self.master_function, self.child_function, args=(self.controller_function, self.minion_function,
self.controller_timeout, self.sleep_time), self.controller_timeout, self.sleep_time),
kwargs=kwargs, kwargs=kwargs,
) )
self.proc.start() self.proc.start()
def join(self): def join(self):
"""Wait for the child process to finish, and return its exit code.""" """Wait for the minion process to finish, and return its exit code."""
self.proc.join() self.proc.join()
return self.proc.exitcode return self.proc.exitcode
@staticmethod @staticmethod
def _set_up_and_run_child_function( def _set_up_and_run_minion_function(
tty_name, stdout_fd, stderr_fd, ready, child_function, **kwargs): tty_name, stdout_fd, stderr_fd, ready, minion_function, **kwargs):
"""Child process wrapper for PseudoShell. """Minion process wrapper for PseudoShell.
Handles the mechanics of setting up a PTY, then calls Handles the mechanics of setting up a PTY, then calls
``child_function``. ``minion_function``.
""" """
# new process group, like a command or pipeline launched by a shell # new process group, like a command or pipeline launched by a shell
@ -266,45 +267,45 @@ def _set_up_and_run_child_function(
if kwargs.get("debug"): if kwargs.get("debug"):
sys.stderr.write( sys.stderr.write(
"child: stdin.isatty(): %s\n" % sys.stdin.isatty()) "minion: stdin.isatty(): %s\n" % sys.stdin.isatty())
# tell the parent that we're really running # tell the parent that we're really running
if kwargs.get("debug"): if kwargs.get("debug"):
sys.stderr.write("child: ready!\n") sys.stderr.write("minion: ready!\n")
ready.value = True ready.value = True
try: try:
child_function(**kwargs) minion_function(**kwargs)
except BaseException: except BaseException:
traceback.print_exc() traceback.print_exc()
@staticmethod @staticmethod
def _set_up_and_run_master_function( def _set_up_and_run_controller_function(
master_function, child_function, controller_timeout, sleep_time, controller_function, minion_function, controller_timeout,
**kwargs): sleep_time, **kwargs):
"""Set up a pty, spawn a child process, and execute master_function. """Set up a pty, spawn a minion process, execute controller_function.
Handles the mechanics of setting up a PTY, then calls Handles the mechanics of setting up a PTY, then calls
``master_function``. ``controller_function``.
""" """
os.setsid() # new session; this process is the controller os.setsid() # new session; this process is the controller
master_fd, child_fd = os.openpty() controller_fd, minion_fd = os.openpty()
pty_name = os.ttyname(child_fd) pty_name = os.ttyname(minion_fd)
# take controlling terminal # take controlling terminal
pty_fd = os.open(pty_name, os.O_RDWR) pty_fd = os.open(pty_name, os.O_RDWR)
os.close(pty_fd) os.close(pty_fd)
ready = multiprocessing.Value('i', False) ready = multiprocessing.Value('i', False)
child_process = multiprocessing.Process( minion_process = multiprocessing.Process(
target=PseudoShell._set_up_and_run_child_function, target=PseudoShell._set_up_and_run_minion_function,
args=(pty_name, sys.stdout.fileno(), sys.stderr.fileno(), args=(pty_name, sys.stdout.fileno(), sys.stderr.fileno(),
ready, child_function), ready, minion_function),
kwargs=kwargs, kwargs=kwargs,
) )
child_process.start() minion_process.start()
# wait for subprocess to be running and connected. # wait for subprocess to be running and connected.
while not ready.value: while not ready.value:
@ -315,30 +316,31 @@ def _set_up_and_run_master_function(
sys.stderr.write("pid: %d\n" % os.getpid()) sys.stderr.write("pid: %d\n" % os.getpid())
sys.stderr.write("pgid: %d\n" % os.getpgrp()) sys.stderr.write("pgid: %d\n" % os.getpgrp())
sys.stderr.write("sid: %d\n" % os.getsid(0)) sys.stderr.write("sid: %d\n" % os.getsid(0))
sys.stderr.write("tcgetpgrp: %d\n" % os.tcgetpgrp(master_fd)) sys.stderr.write("tcgetpgrp: %d\n" % os.tcgetpgrp(controller_fd))
sys.stderr.write("\n") sys.stderr.write("\n")
child_pgid = os.getpgid(child_process.pid) minion_pgid = os.getpgid(minion_process.pid)
sys.stderr.write("child pid: %d\n" % child_process.pid) sys.stderr.write("minion pid: %d\n" % minion_process.pid)
sys.stderr.write("child pgid: %d\n" % child_pgid) sys.stderr.write("minion pgid: %d\n" % minion_pgid)
sys.stderr.write("child sid: %d\n" % os.getsid(child_process.pid)) sys.stderr.write(
"minion sid: %d\n" % os.getsid(minion_process.pid))
sys.stderr.write("\n") sys.stderr.write("\n")
sys.stderr.flush() sys.stderr.flush()
# set up master to ignore SIGTSTP, like a shell # set up controller to ignore SIGTSTP, like a shell
signal.signal(signal.SIGTSTP, signal.SIG_IGN) signal.signal(signal.SIGTSTP, signal.SIG_IGN)
# call the master function once the child is ready # call the controller function once the minion is ready
try: try:
controller = ProcessController( controller = ProcessController(
child_process.pid, master_fd, debug=kwargs.get("debug")) minion_process.pid, controller_fd, debug=kwargs.get("debug"))
controller.timeout = controller_timeout controller.timeout = controller_timeout
controller.sleep_time = sleep_time controller.sleep_time = sleep_time
error = master_function(child_process, controller, **kwargs) error = controller_function(minion_process, controller, **kwargs)
except BaseException: except BaseException:
error = 1 error = 1
traceback.print_exc() traceback.print_exc()
child_process.join() minion_process.join()
# return whether either the parent or child failed # return whether either the parent or minion failed
return error or child_process.exitcode return error or minion_process.exitcode

42
lib/spack/spack/test/llnl/util/tty/log.py
View File

@ -111,7 +111,7 @@ def test_log_subproc_and_echo_output_capfd(capfd, tmpdir):
# Tests below use a pseudoterminal to test llnl.util.tty.log # Tests below use a pseudoterminal to test llnl.util.tty.log
# #
def simple_logger(**kwargs): def simple_logger(**kwargs):
"""Mock logger (child) process for testing log.keyboard_input.""" """Mock logger (minion) process for testing log.keyboard_input."""
def handler(signum, frame): def handler(signum, frame):
running[0] = False running[0] = False
signal.signal(signal.SIGUSR1, handler) signal.signal(signal.SIGUSR1, handler)
@ -125,7 +125,7 @@ def handler(signum, frame):
def mock_shell_fg(proc, ctl, **kwargs): def mock_shell_fg(proc, ctl, **kwargs):
"""PseudoShell master function for test_foreground_background.""" """PseudoShell controller function for test_foreground_background."""
ctl.fg() ctl.fg()
ctl.status() ctl.status()
ctl.wait_enabled() ctl.wait_enabled()
@ -134,7 +134,7 @@ def mock_shell_fg(proc, ctl, **kwargs):
def mock_shell_fg_no_termios(proc, ctl, **kwargs): def mock_shell_fg_no_termios(proc, ctl, **kwargs):
"""PseudoShell master function for test_foreground_background.""" """PseudoShell controller function for test_foreground_background."""
ctl.fg() ctl.fg()
ctl.status() ctl.status()
ctl.wait_disabled_fg() ctl.wait_disabled_fg()
@ -143,7 +143,7 @@ def mock_shell_fg_no_termios(proc, ctl, **kwargs):
def mock_shell_bg(proc, ctl, **kwargs): def mock_shell_bg(proc, ctl, **kwargs):
"""PseudoShell master function for test_foreground_background.""" """PseudoShell controller function for test_foreground_background."""
ctl.bg() ctl.bg()
ctl.status() ctl.status()
ctl.wait_disabled() ctl.wait_disabled()
@ -152,7 +152,7 @@ def mock_shell_bg(proc, ctl, **kwargs):
def mock_shell_tstp_cont(proc, ctl, **kwargs): def mock_shell_tstp_cont(proc, ctl, **kwargs):
"""PseudoShell master function for test_foreground_background.""" """PseudoShell controller function for test_foreground_background."""
ctl.tstp() ctl.tstp()
ctl.wait_stopped() ctl.wait_stopped()
@ -163,7 +163,7 @@ def mock_shell_tstp_cont(proc, ctl, **kwargs):
def mock_shell_tstp_tstp_cont(proc, ctl, **kwargs): def mock_shell_tstp_tstp_cont(proc, ctl, **kwargs):
"""PseudoShell master function for test_foreground_background.""" """PseudoShell controller function for test_foreground_background."""
ctl.tstp() ctl.tstp()
ctl.wait_stopped() ctl.wait_stopped()
@ -177,7 +177,7 @@ def mock_shell_tstp_tstp_cont(proc, ctl, **kwargs):
def mock_shell_tstp_tstp_cont_cont(proc, ctl, **kwargs): def mock_shell_tstp_tstp_cont_cont(proc, ctl, **kwargs):
"""PseudoShell master function for test_foreground_background.""" """PseudoShell controller function for test_foreground_background."""
ctl.tstp() ctl.tstp()
ctl.wait_stopped() ctl.wait_stopped()
@ -194,7 +194,7 @@ def mock_shell_tstp_tstp_cont_cont(proc, ctl, **kwargs):
def mock_shell_bg_fg(proc, ctl, **kwargs): def mock_shell_bg_fg(proc, ctl, **kwargs):
"""PseudoShell master function for test_foreground_background.""" """PseudoShell controller function for test_foreground_background."""
ctl.bg() ctl.bg()
ctl.status() ctl.status()
ctl.wait_disabled() ctl.wait_disabled()
@ -207,7 +207,7 @@ def mock_shell_bg_fg(proc, ctl, **kwargs):
def mock_shell_bg_fg_no_termios(proc, ctl, **kwargs): def mock_shell_bg_fg_no_termios(proc, ctl, **kwargs):
"""PseudoShell master function for test_foreground_background.""" """PseudoShell controller function for test_foreground_background."""
ctl.bg() ctl.bg()
ctl.status() ctl.status()
ctl.wait_disabled() ctl.wait_disabled()
@ -220,7 +220,7 @@ def mock_shell_bg_fg_no_termios(proc, ctl, **kwargs):
def mock_shell_fg_bg(proc, ctl, **kwargs): def mock_shell_fg_bg(proc, ctl, **kwargs):
"""PseudoShell master function for test_foreground_background.""" """PseudoShell controller function for test_foreground_background."""
ctl.fg() ctl.fg()
ctl.status() ctl.status()
ctl.wait_enabled() ctl.wait_enabled()
@ -233,7 +233,7 @@ def mock_shell_fg_bg(proc, ctl, **kwargs):
def mock_shell_fg_bg_no_termios(proc, ctl, **kwargs): def mock_shell_fg_bg_no_termios(proc, ctl, **kwargs):
"""PseudoShell master function for test_foreground_background.""" """PseudoShell controller function for test_foreground_background."""
ctl.fg() ctl.fg()
ctl.status() ctl.status()
ctl.wait_disabled_fg() ctl.wait_disabled_fg()
@ -299,7 +299,7 @@ def test_foreground_background(test_fn, termios_on_or_off, tmpdir):
def synchronized_logger(**kwargs): def synchronized_logger(**kwargs):
"""Mock logger (child) process for testing log.keyboard_input. """Mock logger (minion) process for testing log.keyboard_input.
This logger synchronizes with the parent process to test that 'v' can This logger synchronizes with the parent process to test that 'v' can
toggle output. It is used in ``test_foreground_background_output`` below. toggle output. It is used in ``test_foreground_background_output`` below.
@ -330,7 +330,7 @@ def handler(signum, frame):
def mock_shell_v_v(proc, ctl, **kwargs): def mock_shell_v_v(proc, ctl, **kwargs):
"""PseudoShell master function for test_foreground_background_output.""" """Controller function for test_foreground_background_output."""
write_lock = kwargs["write_lock"] write_lock = kwargs["write_lock"]
v_lock = kwargs["v_lock"] v_lock = kwargs["v_lock"]
@ -357,7 +357,7 @@ def mock_shell_v_v(proc, ctl, **kwargs):
def mock_shell_v_v_no_termios(proc, ctl, **kwargs): def mock_shell_v_v_no_termios(proc, ctl, **kwargs):
"""PseudoShell master function for test_foreground_background_output.""" """Controller function for test_foreground_background_output."""
write_lock = kwargs["write_lock"] write_lock = kwargs["write_lock"]
v_lock = kwargs["v_lock"] v_lock = kwargs["v_lock"]
@ -395,9 +395,9 @@ def test_foreground_background_output(
shell = PseudoShell(test_fn, synchronized_logger) shell = PseudoShell(test_fn, synchronized_logger)
log_path = str(tmpdir.join("log.txt")) log_path = str(tmpdir.join("log.txt"))
# Locks for synchronizing with child # Locks for synchronizing with minion
write_lock = multiprocessing.Lock() # must be held by child to write write_lock = multiprocessing.Lock() # must be held by minion to write
v_lock = multiprocessing.Lock() # held while master is in v mode v_lock = multiprocessing.Lock() # held while controller is in v mode
with termios_on_or_off(): with termios_on_or_off():
shell.start( shell.start(
@ -423,16 +423,16 @@ def test_foreground_background_output(
with open(log_path) as log: with open(log_path) as log:
log = log.read().strip().split("\n") log = log.read().strip().split("\n")
# Master and child process coordinate with locks such that the child # Controller and minion process coordinate with locks such that the minion
# writes "off" when echo is off, and "on" when echo is on. The # writes "off" when echo is off, and "on" when echo is on. The
# output should contain mostly "on" lines, but may contain an "off" # output should contain mostly "on" lines, but may contain an "off"
# or two. This is because the master toggles echo by sending "v" on # or two. This is because the controller toggles echo by sending "v" on
# stdin to the child, but this is not synchronized with our locks. # stdin to the minion, but this is not synchronized with our locks.
# It's good enough for a test, though. We allow at most 2 "off"'s in # It's good enough for a test, though. We allow at most 2 "off"'s in
# the output to account for the race. # the output to account for the race.
assert ( assert (
['forced output', 'on'] == uniq(output) or ['forced output', 'on'] == uniq(output) or
output.count("off") <= 2 # if master_fd is a bit slow output.count("off") <= 2 # if controller_fd is a bit slow
) )
# log should be off for a while, then on, then off # log should be off for a while, then on, then off