""" =============================================================== Run any Python script in a thread, with arguments passed in, and output (including prints) redirected to an object. Mergeall's scripts can be run by subprocess.Popen on Android (and are, in the original tkinter GUI), but using a thread sidesteps the child-process killing potential in Android 12+. This is an alternative to running scripts in subprocesses, coding+importing main() functions for all scripts' __main__ logic, and changing all print()s to use a custom object. This is also brittle: it resets global system items for thread duration, and will fail if any other parallel code depends on these. Use only if this constraint is known to be satisfied. See also the corresponding modules for services and processes. =============================================================== """ import sys, threading, builtins, os, queue, time, traceback, io def message(msg=''): """ For prints while thread reset active. Testing only - comment out stderr lines in next function to use Caution - don't send output to logcat in Android release builds """ sys.stderr.write(msg + '\n') sys.stderr.flush() def run_script_thread_wrapper( mainfilepath, # path to the script file to run in thread (rel|abs) argslist, # command-line arguments list for the script stdouterr, # where to route stdout (and perhaps stderr) postop=None, # a no-arg callable to run in the thread after script stdin=None, # a string to use to thread's stdin, '\n' for multiline trace=False, # if True, don't redirect stderr, display state there chdir=True): # if True, change to folder of script (for rels) """ Run a script file's code in a thread, with system globals reset. Alternative to importing script's main function (in '.' or a package folder), for scripts whose main logic isn't coded as a function. Assumes thread's code is only user of system globals while it runs; the os.chdir call seems the most dangerous - disable vis arg chdir. Note that module imports in the script code run in the thread go into the global sys.modules, and hence are shared with the calling thread. The spawner can mod script behavior by changing global/shared modules, and the script will import its modules just once for all thread runs. By contrast, processes import anew into their own state on each run. On Android daemon=True is crucial, to allow app to close even if the thread is running; else, the UI thread goes away leaving a blank screen. trace is just for debugging - thread's output should be just its report. For the same reason, stdout/err file-descriptor dups are ruled out here. Subtle: mergeall uses some state globals that must be reset in main.py per action run, because scripts aren't run as new processes. Subtle: mergeall.py resets buitins.print for output buffers in its PyInstaller frozen executables on Windows and Linux. The reset is moot here (mergeall is run as source code embedded in a frozen exe in PCs, and its output is intercepted here), but the mergeall reset code still fires in this context, so print must be restored here, else the prints in the app's trace() get two 'flush' arguments and fail with excs. Subtler: builtin.print must be restored here _before_ stdouterr.close. Else, the close triggers the Flusher's on_eof callback; which runs the app's on_script_exit immediately in the action thread and same process; and on_script_exit is @mainthread which means it's scheduled for later run on the GUI queue; but its script_running assignment may have already run to enable app prints when the Clock callback registered for the Logs tab's WATCH is fired; which calls enable_logs_text_hscroll(); which runs a trace() that prints and raises an exception for double flush=True if the builtin print is not yet reset; but only if WATCH is on for actions run in thread mode in the app's Windows and Linux frozen exes. Really. Subtlest: even with the mod just noted, print() exceptions were still seen in the Windows exe for duplicate flush=True args on tabs-switch callbacks. There remains an unknown timing issue. But as a total fix, mergeall/mergeall.py was modified to avoid adding a flush=True if one already exists in the **kargs passed in from main.py's trace(). The excs' cause remains TBD, but such is life with threads and deferred callbacks. """ def showstate(): sys.stderr.write( '=' * 4 + '\n' + os.getcwd() + '\n' + str(sys.path) + '\n' + str(os.listdir('.')) + '\n' + '=' * 4 + '\n') # where/what to run mainfiledir, mainfile = os.path.split(mainfilepath) mainfileabs = os.path.abspath(mainfiledir) # save global state savename = builtins.__name__ saveprint = builtins.print # mergeall.py sets print for win/lin exes! saveargv = sys.argv saveout = sys.stdout saveerr = sys.stderr savein = sys.stdin savepath = sys.path.copy() savedir = os.getcwd() if trace: showstate() # change global state builtins.__name__ = '__main__' sys.argv = [mainfileabs] + argslist # command-line arguments sys.stdout = stdouterr # prints, writes (or reset .print) sys.path = [mainfileabs] + sys.path # module imports ('.' fails, moot?) if not trace: sys.stderr = stdouterr # catch/show exceptions in script if chdir: os.chdir(mainfiledir) # relative-path files (iff okay) if stdin != None: sys.stdin = io.StringIO(stdin) # provide canned script input() if trace: showstate() def runner(): # in diff thread, but same pvm, process, and scopes try: pycode = open(mainfile, 'r', encoding='utf8').read() globalscope = {} exec(pycode, globalscope) if postop != None: postop() except SystemExit: # mergeall -report mode ends with sys.exit(0) # if uncaught: finally+reraise (but moot here: # run in a thread, and exit signaled by Flusher) pass except Exception as E: # other exceptions in script code: to logfile|queue # testing only: message(str(E)) traceback.print_exc() # to logfile finally: # exception or not # restore global state after code exit builtins.__name__ = savename builtins.print = saveprint sys.argv = saveargv sys.stdout = saveout sys.stderr = saveerr sys.stdin = savein sys.path = savepath os.chdir(savedir) # send eof signal to listener (post print restore!) stdouterr.close() # spawn the thread thread = threading.Thread(target=runner) # run this in a new pthread thread.daemon = True # exit app if this thread still running thread.start() # invoke start() => run() => target return thread class Queuer: """ Accept thread stdout writes (and, by proxy, prints). Make their text available on a blocking line queue. May include stderr writes and print(file=sys.stderr)s if trace=False in the thread spawner function. Queue always has lines ending in \n' or eof marker, even if the last line was not \n-terminated. The user of this clas should take care when writing its queued lines to a file or stream, because Unicode errors are always possible: see Flusher ahead. In this app, we no longer care, because this class is not used (so far?). """ eofsignal = None def __init__(self): self.linequeue = queue.SimpleQueue() # fifo, unbounded, py3.7+ self.linesofar = '' def write(self, text): while text: eoln = text.find('\n') # 0..N \n-terminated lines if eoln == -1: self.linesofar += text text = '' else: line = text[:eoln+1] text = text[eoln+1:] self.linequeue.put(self.linesofar + line) self.linesofar = '' def flush(self): pass def close(self): if self.linesofar: # remaining text? self.linequeue.put(self.linesofar + '\n') # make line: add \n self.linequeue.put(self.eofsignal) # note thread exit def readline(self): return self.linequeue.get(block=True) class Flusher: """ Wrap an output file to auto-flush all writes, so that output appears immediately in the file to be seen by file watchers. Also handles bad Unicode chanracters in output lines. The on_eof argument can be used as an end-of-thread callback signal, and can be any callable in the spawner. This works here because the thread runs in the same, single process. Alt: force unbuffered mode in file via os calls; portable? On write exceptions due to Unicode errors, retry with ascii() to escape all non-ascii chars. Else, a bad character in a filename can crash the run due to output-stream write errors. See the same-named class in run_script_as_service.py for more. [1.2.0] mergeall.py calls isatty() to check if it should display help text to a console, in the unlikely event of a command-line error (which actually happened before relative-path verification was tightened up in the GUI); define here to avoid exception. This was an issue only here for threads, but not for services: run_script_as_service.py gets isatty() by proxy via __getattr__. """ def __init__(self, fileobj, on_eof=(lambda: None)): self.fileobj = fileobj self.on_eof = on_eof # in same process def write(self, text): try: self.fileobj.write(text) # even utf8 can fail except UnicodeEncodeError: # retry as ascii+\escapes try: putback = '\n' if text[-1] == '\n' else '' if putback: text = text[:-1] self.fileobj.write(ascii(text)[1:-1] + putback) except: self.fileobj.write('\n') self.fileobj.flush() # to file for tailers def flush(self): pass # moot: write() already flushes def isatty(self): # [1.2.0] on mergeall.py errors return False # don't show help text (or exc) def close(self): # caller closes fileobj self.on_eof() # script-code exit signal # __getattr__ not required def consumer(pipe, pausesecs=0.10, batchlimit=30): """ In a gui, do this in a timer-callback event so runs in main ui thread. Tweak params for speed, and to avoid blocking the gui thread too much. """ while True: time.sleep(pausesecs) # on timer event try: line = pipe.linequeue.get(block=False) except queue.Empty: continue # back to top of timer loop # process a batch of lines batchnum = 0 while True: if line == pipe.eofsignal: message('') return # end of consuming else: batchnum += 1 message('' + line.rstrip('\n')) # refresh gui here if batchnum == batchlimit: break # back to top of timer loop elif pipe.linequeue.empty(): break # back to top of timer loop else: line = pipe.linequeue.get(block=False) # back to top of batch loop if __name__ == '__main__': # when run as script only: testing if True: print() pipe = Queuer() thread = run_script_thread_wrapper( 'mergeall/mergeall.py', ['..', '..', '-report', '-skipcruft'], pipe) consumer(pipe) thread.join() if True: print() pipe = Queuer() thread = run_script_thread_wrapper( 'mergeall/diffall.py', ['..', '..', '-skipcruft'], pipe) consumer(pipe) thread.join() print() print('Bye') # stdout restored here """ ------------------------------------------------------------------ Validation: ~/Desktop/DEV-BD/apps/PC-Phone-USB-Sync$ diff da-thread da-direct 1d0 < 40c39 < Runtime hrs:mins:secs = 0:0:0.03 --- > Runtime hrs:mins:secs = 0:0:0.01 45,47d43 < < < Bye ~/Desktop/DEV-BD/apps/PC-Phone-USB-Sync$ diff ma-thread ma-direct 1d0 < 8c7 < Phase runtime: 0.003291906000000011 --- > Phase runtime: 0.002886950000000006 40,42d38 < < < Bye ------------------------------------------------------------------ """