File: PC-Phone USB Sync/code/PC-Phone USB Sync--source/run_script_as_thread.py
"""
===============================================================
Run any Python script in a thread, with arguments passed
in, and output (including prints) redirected to an object.
Mergeall's scripts can be run by subprocess.Popen on Android
(and are, in the original tkinter GUI), but using a thread
sidesteps the child-process killing potential in Android 12+.
This is an alternative to running scripts in subprocesses,
coding+importing main() functions for all scripts' __main__
logic, and changing all print()s to use a custom object.
This is also brittle: it resets global system items for thread
duration, and will fail if any other parallel code depends on
these. Use only if this constraint is known to be satisfied.
See also the corresponding modules for services and processes.
===============================================================
"""
import sys, threading, builtins, os, queue, time, traceback, io
def message(msg=''):
"""
For prints while thread reset active.
Testing only - comment out stderr lines in next function to use
Caution - don't send output to logcat in Android release builds
"""
sys.stderr.write(msg + '\n')
sys.stderr.flush()
def run_script_thread_wrapper(
mainfilepath, # path to the script file to run in thread (rel|abs)
argslist, # command-line arguments list for the script
stdouterr, # where to route stdout (and perhaps stderr)
postop=None, # a no-arg callable to run in the thread after script
stdin=None, # a string to use to thread's stdin, '\n' for multiline
trace=False, # if True, don't redirect stderr, display state there
chdir=True): # if True, change to folder of script (for rels)
"""
Run a script file's code in a thread, with system globals reset.
Alternative to importing script's main function (in '.' or a package
folder), for scripts whose main logic isn't coded as a function.
Assumes thread's code is only user of system globals while it runs;
the os.chdir call seems the most dangerous - disable vis arg chdir.
Note that module imports in the script code run in the thread go into
the global sys.modules, and hence are shared with the calling thread.
The spawner can mod script behavior by changing global/shared modules,
and the script will import its modules just once for all thread runs.
By contrast, processes import anew into their own state on each run.
On Android daemon=True is crucial, to allow app to close even if the
thread is running; else, the UI thread goes away leaving a blank screen.
trace is just for debugging - thread's output should be just its report.
For the same reason, stdout/err file-descriptor dups are ruled out here.
Subtle: mergeall uses some state globals that must be reset in main.py
per action run, because scripts aren't run as new processes.
Subtle: mergeall.py resets buitins.print for output buffers in its
PyInstaller frozen executables on Windows and Linux. The reset is moot
here (mergeall is run as source code embedded in a frozen exe in PCs,
and its output is intercepted here), but the mergeall reset code still
fires in this context, so print must be restored here, else the prints
in the app's trace() get two 'flush' arguments and fail with excs.
Subtler: builtin.print must be restored here _before_ stdouterr.close.
Else, the close triggers the Flusher's on_eof callback; which runs the
app's on_script_exit immediately in the action thread and same process;
and on_script_exit is @mainthread which means it's scheduled for later
run on the GUI queue; but its script_running assignment may have already
run to enable app prints when the Clock callback registered for the Logs
tab's WATCH is fired; which calls enable_logs_text_hscroll(); which runs
a trace() that prints and raises an exception for double flush=True if
the builtin print is not yet reset; but only if WATCH is on for actions
run in thread mode in the app's Windows and Linux frozen exes. Really.
Subtlest: even with the mod just noted, print() exceptions were still
seen in the Windows exe for duplicate flush=True args on tabs-switch
callbacks. There remains an unknown timing issue. But as a total fix,
mergeall/mergeall.py was modified to avoid adding a flush=True if one
already exists in the **kargs passed in from main.py's trace(). The excs'
cause remains TBD, but such is life with threads and deferred callbacks.
"""
def showstate():
sys.stderr.write(
'=' * 4 + '\n' +
os.getcwd() + '\n' +
str(sys.path) + '\n' +
str(os.listdir('.')) + '\n' +
'=' * 4 + '\n')
# where/what to run
mainfiledir, mainfile = os.path.split(mainfilepath)
mainfileabs = os.path.abspath(mainfiledir)
# save global state
savename = builtins.__name__
saveprint = builtins.print # mergeall.py sets print for win/lin exes!
saveargv = sys.argv
saveout = sys.stdout
saveerr = sys.stderr
savein = sys.stdin
savepath = sys.path.copy()
savedir = os.getcwd()
if trace: showstate()
# change global state
builtins.__name__ = '__main__'
sys.argv = [mainfileabs] + argslist # command-line arguments
sys.stdout = stdouterr # prints, writes (or reset .print)
sys.path = [mainfileabs] + sys.path # module imports ('.' fails, moot?)
if not trace:
sys.stderr = stdouterr # catch/show exceptions in script
if chdir:
os.chdir(mainfiledir) # relative-path files (iff okay)
if stdin != None:
sys.stdin = io.StringIO(stdin) # provide canned script input()
if trace: showstate()
def runner():
# in diff thread, but same pvm, process, and scopes
try:
pycode = open(mainfile, 'r', encoding='utf8').read()
globalscope = {}
exec(pycode, globalscope)
if postop != None: postop()
except SystemExit:
# mergeall -report mode ends with sys.exit(0)
# if uncaught: finally+reraise (but moot here:
# run in a thread, and exit signaled by Flusher)
pass
except Exception as E:
# other exceptions in script code: to logfile|queue
# testing only: message(str(E))
traceback.print_exc() # to logfile
finally:
# exception or not
# restore global state after code exit
builtins.__name__ = savename
builtins.print = saveprint
sys.argv = saveargv
sys.stdout = saveout
sys.stderr = saveerr
sys.stdin = savein
sys.path = savepath
os.chdir(savedir)
# send eof signal to listener (post print restore!)
stdouterr.close()
# spawn the thread
thread = threading.Thread(target=runner) # run this in a new pthread
thread.daemon = True # exit app if this thread still running
thread.start() # invoke start() => run() => target
return thread
class Queuer:
"""
Accept thread stdout writes (and, by proxy, prints).
Make their text available on a blocking line queue.
May include stderr writes and print(file=sys.stderr)s
if trace=False in the thread spawner function.
Queue always has lines ending in \n' or eof marker,
even if the last line was not \n-terminated.
The user of this clas should take care when writing its
queued lines to a file or stream, because Unicode errors
are always possible: see Flusher ahead. In this app, we
no longer care, because this class is not used (so far?).
"""
eofsignal = None
def __init__(self):
self.linequeue = queue.SimpleQueue() # fifo, unbounded, py3.7+
self.linesofar = ''
def write(self, text):
while text:
eoln = text.find('\n') # 0..N \n-terminated lines
if eoln == -1:
self.linesofar += text
text = ''
else:
line = text[:eoln+1]
text = text[eoln+1:]
self.linequeue.put(self.linesofar + line)
self.linesofar = ''
def flush(self):
pass
def close(self):
if self.linesofar: # remaining text?
self.linequeue.put(self.linesofar + '\n') # make line: add \n
self.linequeue.put(self.eofsignal) # note thread exit
def readline(self):
return self.linequeue.get(block=True)
class Flusher:
"""
Wrap an output file to auto-flush all writes, so that output
appears immediately in the file to be seen by file watchers.
Also handles bad Unicode chanracters in output lines.
The on_eof argument can be used as an end-of-thread callback
signal, and can be any callable in the spawner. This works
here because the thread runs in the same, single process.
Alt: force unbuffered mode in file via os calls; portable?
On write exceptions due to Unicode errors, retry with ascii()
to escape all non-ascii chars. Else, a bad character in a
filename can crash the run due to output-stream write errors.
See the same-named class in run_script_as_service.py for more.
[1.2.0] mergeall.py calls isatty() to check if it should display
help text to a console, in the unlikely event of a command-line
error (which actually happened before relative-path verification
was tightened up in the GUI); define here to avoid exception.
This was an issue only here for threads, but not for services:
run_script_as_service.py gets isatty() by proxy via __getattr__.
"""
def __init__(self, fileobj, on_eof=(lambda: None)):
self.fileobj = fileobj
self.on_eof = on_eof # in same process
def write(self, text):
try:
self.fileobj.write(text) # even utf8 can fail
except UnicodeEncodeError: # retry as ascii+\escapes
try:
putback = '\n' if text[-1] == '\n' else ''
if putback: text = text[:-1]
self.fileobj.write(ascii(text)[1:-1] + putback)
except:
self.fileobj.write('<Cannot write output text>\n')
self.fileobj.flush() # to file for tailers
def flush(self):
pass # moot: write() already flushes
def isatty(self): # [1.2.0] on mergeall.py errors
return False # don't show help text (or exc)
def close(self): # caller closes fileobj
self.on_eof() # script-code exit signal
# __getattr__ not required
def consumer(pipe, pausesecs=0.10, batchlimit=30):
"""
In a gui, do this in a timer-callback event so runs in main ui thread.
Tweak params for speed, and to avoid blocking the gui thread too much.
"""
while True:
time.sleep(pausesecs)
# on timer event
try:
line = pipe.linequeue.get(block=False)
except queue.Empty:
continue # back to top of timer loop
# process a batch of lines
batchnum = 0
while True:
if line == pipe.eofsignal:
message('')
return # end of consuming
else:
batchnum += 1
message('' + line.rstrip('\n'))
# refresh gui here
if batchnum == batchlimit:
break # back to top of timer loop
elif pipe.linequeue.empty():
break # back to top of timer loop
else:
line = pipe.linequeue.get(block=False) # back to top of batch loop
if __name__ == '__main__':
# when run as script only: testing
if True:
print()
pipe = Queuer()
thread = run_script_thread_wrapper(
'mergeall/mergeall.py',
['..', '..', '-report', '-skipcruft'],
pipe)
consumer(pipe)
thread.join()
if True:
print()
pipe = Queuer()
thread = run_script_thread_wrapper(
'mergeall/diffall.py',
['..', '..', '-skipcruft'],
pipe)
consumer(pipe)
thread.join()
print()
print('Bye') # stdout restored here
"""
------------------------------------------------------------------
Validation:
~/Desktop/DEV-BD/apps/PC-Phone-USB-Sync$ diff da-thread da-direct
1d0
<
40c39
< Runtime hrs:mins:secs = 0:0:0.03
---
> Runtime hrs:mins:secs = 0:0:0.01
45,47d43
<
<
< Bye
~/Desktop/DEV-BD/apps/PC-Phone-USB-Sync$ diff ma-thread ma-direct
1d0
<
8c7
< Phase runtime: 0.003291906000000011
---
> Phase runtime: 0.002886950000000006
40,42d38
<
<
< Bye
------------------------------------------------------------------
"""