This is the mail archive of the
cygwin
mailing list for the Cygwin project.
cygwin python: subprocess.Popen().stdout + threads == hang
- From: "Charles Wilson" <cygwin at cwilson dot fastmail dot fm>
- To: cygwin at cygwin dot com
- Date: Thu, 24 Jul 2008 12:11:46 -0400
- Subject: cygwin python: subprocess.Popen().stdout + threads == hang
Ran into a weird issue using python on cygwin. When trying to use the
stdout handle from a subprocess spawned using the subprocess.Popen
class, it hangs when accessed via a thread. This behavior does /not/
occur on other platforms -- nor when using os.popen from threads on
cygwin. Nor when using subprocess.Popen from cygwin without threads.
I know I could avoid this by using subprocess.Popen(...).communicate(),
but in my /real/ application, I want line-by-line access to the sub's
output, whereas communicate() gobbles the entire output and returns a
list of strings for stdout, and for stderr. From the help text:
communicate() returns a tuple (stdout, stderr).
Note: The data read is buffered in memory, so do not use this
method if the data size is large or unlimited.
So, I don't want to do /that/.
I've attached a test case to demonstrate the issue. I was hoping (a)
somebody has seen this before, and has a simple solution for me, or (b)
the python maintainer (Jason?) could take a look, and maybe raise a bug
report upstream.
I'm using 2.5.1-2, but I see the same behavior on cygwin using 2.5.2.
When run on native windows, this test case requires Fping.exe
(http://www.kwakkelflap.com/fping.html) because -- at least on Vista --
the native ping is b0rked. When native ping is called on Vista from a
threaded context, if a host is unreachable, it reports summary info for
the previously-accessed reachable host. Very strange -- but not related
to the problem at hand. For cygwin, of course, the test case uses
/usr/bin/ping.
Test case --help output:
Usage: ./testprog.py [htsS]
Tests os.popen vs. subprocess.Popen for various platforms
On native windows: requires Fping.exe
(http://www.kwakkelflap.com/fping.html)
In operation, this program 'pings' ten IP addresses in the current
network, including localhost. It is used to demonstrate that the
.stdout member of subprocess.Popen() does not work on Cygwin when
using threads
-h, --help print this message
-t, --threads use threads (default is seqential)
-s, --subprocess use subprocess.Popen().stdout (default is os.popen)
-S, --shell when using subprocess.Popen(), execute the target
using
the shell (/bin/sh or cmd.exe), just like os.popen
does.
default for subprocess.Popen is to use an
execv-style list.
Behavior: native windows cygwin
unix
<no args> (os.popen, sequential) OK OK
OK
-t (os.popen, threaded) OK OK
OK
-s -S (subprocess, sequential, via shell) OK OK
OK
-s (subprocess, sequential, execv) OK OK
OK
-t -s -S (subprocess, threaded, via shell) OK hangs
OK
-t -s (subprocess, threaded, execv) OK hangs
OK
#!/usr/bin/python
"""
Usage: %s [htsS]
Tests os.popen vs. subprocess.Popen for various platforms
On native windows: requires Fping.exe (http://www.kwakkelflap.com/fping.html)
In operation, this program 'pings' ten IP addresses in the current
network, including localhost. It is used to demonstrate that the
.stdout member of subprocess.Popen() does not work on Cygwin when
using threads
-h, --help print this message
-t, --threads use threads (default is seqential)
-s, --subprocess use subprocess.Popen().stdout (default is os.popen)
-S, --shell when using subprocess.Popen(), execute the target using
the shell (/bin/sh or cmd.exe), just like os.popen does.
default for subprocess.Popen is to use an execv-style list.
Behavior: native windows cygwin unix
<no args> (os.popen, sequential) OK OK OK
-t (os.popen, threaded) OK OK OK
-s -S (subprocess, sequential, via shell) OK OK OK
-s (subprocess, sequential, execv) OK OK OK
-t -s -S (subprocess, threaded, via shell) OK hangs OK
-t -s (subprocess, threaded, execv) OK hangs OK
"""
import os
import re
import time
import sys
import getopt
from threading import Thread
import socket
import struct
import subprocess
# NOTE: OPT_use_shell is always treated as True if OPT_use_subprocess is False
# (that is, os.popen always uses the shell!)
OPT_use_threads=False
OPT_use_subprocess=False
OPT_use_shell=False
if sys.platform == 'win32':
def ping_cmdline(ip, shell=False):
# windows ping is b0rked on Vista, so we're forced to use a different
# ping program. Can't use cygwin ping, must be a native program.
# Fping from http://www.kwakkelflap.com/fping.html seems pretty
# good, is virus free, and is not b0rked.
if shell:
return "Fping.exe " + ip + " -i -n 2"
return ["Fping.exe ", ip, "-i", "-n", "2"]
elif sys.platform == 'cygwin':
def ping_cmdline(ip, shell=False):
# cygwin's ping is also odd
if shell:
return "ping.exe -q " + ip + " 56 2"
return ["ping.exe", "-q", ip, "56", "2"]
else:
def ping_cmdline(ip, shell=False):
if shell:
return "ping -q -c2 " + ip
return ["ping", "-q", "-c2", ip]
def get_pipe_subprocess_Popen(ip):
global OPT_use_shell
print >>sys.stderr, "Using subprocess.Popen (shell=%s)" % OPT_use_shell
chld_stdin = os.open(os.devnull, os.O_RDONLY);
return subprocess.Popen(ping_cmdline(ip, shell=OPT_use_shell),
shell=OPT_use_shell,
stdin=chld_stdin,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).stdout
def get_pipe_os_popen(ip):
global OPT_use_shell
print >>sys.stderr, "Using os.popen (shell=%s)" % OPT_use_shell
return os.popen(ping_cmdline(ip, shell=OPT_use_shell), "r")
def get_pipe(ip):
global OPT_use_subprocess
if OPT_use_subprocess:
return get_pipe_subprocess_Popen(ip)
else:
return get_pipe_os_popen(ip)
class testit(Thread):
def __init__ (self,ip):
Thread.__init__(self)
self.ip = ip
self.status = -1
def run(self):
pingaling = get_pipe(self.ip)
while 1:
line = pingaling.readline()
if not line: break
igot = re.findall(testit.lifeline,line)
if igot:
self.status = int(igot[0])
if sys.platform == 'win32':
testit.lifeline = re.compile(r"Received = (\d)")
lifeline = re.compile(r"Received = (\d)")
elif sys.platform == 'cygwin':
testit.lifeline = re.compile(r"(\d) packets received")
lifeline = re.compile(r"(\d) packets received")
else:
testit.lifeline = re.compile(r"(\d) received")
lifeline = re.compile(r"(\d) received")
report = ("No response","Partial Response","Alive","Internal Error")
def naive_get_ip_addr():
return socket.gethostbyname(socket.gethostname())
def dottedQuadToNum(ip):
"convert decimal dotted quad string to long integer"
return struct.unpack('!L',socket.inet_aton(ip))[0]
def numToDottedQuad(n):
"convert long int to dotted quad string"
return socket.inet_ntoa(struct.pack('!L',n))
def makeMask(n):
"return a mask of n bits as a long integer"
return (1L<<n)-1
def ipToNetAndHost(ip, maskbits):
"returns tuple (network, host) dotted-quad addresses given IP and mask size"
# (by Greg Jorgensen)
n = dottedQuadToNum(ip)
m = makeMask(maskbits)
host = n & m
net = n - host
return numToDottedQuad(net), numToDottedQuad(host)
def main_threads(hosts):
print >>sys.stderr, "Using threads"
pinglist = []
for ip in hosts:
current = testit(ip)
pinglist.append(current)
current.start()
for pingle in pinglist:
pingle.join()
print "Status from ",pingle.ip,"is",report[pingle.status]
def main_nothreads(hosts):
print >>sys.stderr, "No threads"
for ip in hosts:
pingaling = get_pipe(ip)
while 1:
line = pingaling.readline()
if not line: break
igot = re.findall(lifeline,line)
if igot:
print "Status from ",ip," is ",report[int(igot[0])]
def main(argv=None):
global OPT_use_threads, OPT_use_subprocess, OPT_use_shell
if argv is None:
argv = sys.argv
opts, args = getopt.getopt(argv[1:], "htsS",["help", "threads", "subprocess", "shell"])
for opt, value in opts:
if opt in ('-h', '--help'):
print __doc__ % argv[0]
return 0
if opt in ('-t', '--threads'):
OPT_use_threads = True
if opt in ('-s', '--subprocess'):
OPT_use_subprocess = True
if opt in ('-S', '--shell'):
OPT_use_shell = True
# NOTE: if using os.popen (that is, OPT_use_subprocess is false)
# then we *always* use the shell
if not OPT_use_subprocess:
OPT_use_shell = True
# compute range of ip addresses
myip = naive_get_ip_addr()
(netwk, hst) = ipToNetAndHost(myip, 8)
hstd = dottedQuadToNum(hst)
netwkd = dottedQuadToNum(netwk)
if hstd >= 250:
hostrange = range(245,254)
else:
hostrange = range(hstd - (hstd % 10) + 1,
hstd - (hstd % 10) + 11)
hosts = []
for host in hostrange:
ip = numToDottedQuad(netwkd + host)
hosts.append(ip)
print time.ctime()
if OPT_use_threads:
main_threads(hosts)
else:
main_nothreads(hosts)
print time.ctime()
if __name__ == "__main__":
sys.exit(main())
--
Unsubscribe info: http://cygwin.com/ml/#unsubscribe-simple
Problem reports: http://cygwin.com/problems.html
Documentation: http://cygwin.com/docs.html
FAQ: http://cygwin.com/faq/