diff --git a/Lib/_socket.py b/Lib/_socket.py index 74c756f8b..98a5a98ef 100644 --- a/Lib/_socket.py +++ b/Lib/_socket.py @@ -1130,6 +1130,10 @@ def _finish_closing(self, _): log.debug("Closed child socket %s not yet accepted", child, extra={"sock": self}) child.close() else: + if self.incoming_head is not None: + if self.incoming_head is not _PEER_CLOSED: + self.incoming_head.release() + self.incoming_head = None msgs = [] self.incoming.drainTo(msgs) for msg in msgs: diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 8a2db9508..f3d9b11f1 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -6,6 +6,7 @@ import errno import os import tempfile +import urlparse import unittest TestCase = unittest.TestCase @@ -22,6 +23,26 @@ HOST = test_support.HOST +def _external_https_connection(host, port=443, **kwargs): + proxy = _https_proxy() + if proxy is None: + return httplib.HTTPSConnection(host, port, **kwargs) + proxy_host, proxy_port = proxy + conn = httplib.HTTPSConnection(proxy_host, proxy_port, **kwargs) + conn.set_tunnel(host, port) + return conn + +def _https_proxy(): + for name in ('https_proxy', 'HTTPS_PROXY', 'http_proxy', 'HTTP_PROXY'): + proxy = os.environ.get(name) + if proxy: + if '://' not in proxy: + proxy = '//' + proxy + parsed = urlparse.urlparse(proxy) + if parsed.hostname: + return parsed.hostname, parsed.port or 80 + return None + class FakeSocket: def __init__(self, text, fileclass=StringIO.StringIO, host=None, port=None): self.text = text @@ -827,7 +848,7 @@ def test_networked(self): import ssl test_support.requires('network') with test_support.transient_internet('self-signed.pythontest.net'): - h = httplib.HTTPSConnection('self-signed.pythontest.net', 443) + h = _external_https_connection('self-signed.pythontest.net', 443) with self.assertRaises(ssl.SSLError) as exc_info: h.request('GET', '/') if test_support.is_jython: @@ -840,8 +861,8 @@ def test_networked_noverification(self): test_support.requires('network') with test_support.transient_internet('self-signed.pythontest.net'): context = ssl._create_stdlib_context() - h = httplib.HTTPSConnection('self-signed.pythontest.net', 443, - context=context) + h = _external_https_connection('self-signed.pythontest.net', 443, + context=context) h.request('GET', '/') resp = h.getresponse() self.assertIn('nginx', resp.getheader('server')) @@ -849,9 +870,10 @@ def test_networked_noverification(self): @test_support.system_must_validate_cert def test_networked_trusted_by_default_cert(self): # Default settings: requires a valid cert from a trusted CA + # www.pythontest.net intentionally uses a self-signed certificate. test_support.requires('network') with test_support.transient_internet('www.python.org'): - h = httplib.HTTPSConnection('www.python.org', 443) + h = _external_https_connection('www.python.org', 443) h.request('GET', '/') resp = h.getresponse() content_type = resp.getheader('content-type') @@ -865,7 +887,7 @@ def test_networked_good_cert(self): context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) context.verify_mode = ssl.CERT_REQUIRED context.load_verify_locations(CERT_selfsigned_pythontestdotnet) - h = httplib.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + h = _external_https_connection('self-signed.pythontest.net', 443, context=context) h.request('GET', '/') resp = h.getresponse() server_string = resp.getheader('server') @@ -879,7 +901,7 @@ def test_networked_bad_cert(self): context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) context.verify_mode = ssl.CERT_REQUIRED context.load_verify_locations(CERT_localhost) - h = httplib.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + h = _external_https_connection('self-signed.pythontest.net', 443, context=context) with self.assertRaises(ssl.SSLError) as exc_info: h.request('GET', '/') if test_support.is_jython: @@ -996,6 +1018,13 @@ def create_connection(address, timeout=None, source_address=None): @test_support.reap_threads def test_main(verbose=None): + if test_support.is_jython: + from java.util.logging import Logger, Level + # Netty logs noisy channel-initializer warnings when tests deliberately + # close localhost sockets in error paths. + Logger.getLogger("io.netty.channel.ChannelInitializer").setLevel(Level.SEVERE) + Logger.getLogger("io.netty.util.concurrent.DefaultPromise").setLevel(Level.OFF) + test_support.run_unittest(HeaderTests, OfflineTest, BasicTest, TimeoutTest, HTTPTest, HTTPSTest, SourceAddressTest, TunnelTests) diff --git a/Lib/test/test_os_jy.py b/Lib/test/test_os_jy.py index 640408815..2cb409c10 100644 --- a/Lib/test/test_os_jy.py +++ b/Lib/test/test_os_jy.py @@ -534,6 +534,30 @@ def test_readlink_nonexistent(self): self.assertEqual(cm.exception.filename, nonexistent_file) +class EnvironmentVarGuardTestCase(unittest.TestCase): + + @unittest.skipUnless(getattr(os, '_name', os.name) == 'nt', + "Windows environment names are case-insensitive") + def test_case_insensitive_unset_restore(self): + lower = "jy_envguard_case_test" + upper = "JY_ENVGUARD_CASE_TEST" + had_value = lower in os.environ or upper in os.environ + original_value = os.environ.get(lower) + os.environ[lower] = "original" + try: + with test_support.EnvironmentVarGuard() as env: + env.unset(lower) + env.unset(upper) + self.assertEqual(os.environ.get(lower), "original") + self.assertEqual(os.environ.get(upper), "original") + finally: + if had_value: + os.environ[lower] = original_value + else: + if lower in os.environ: + del os.environ[lower] + + def test_main(): test_support.run_unittest( OSFileTestCase, @@ -545,6 +569,7 @@ def test_main(): SystemTestCase, LinkTestCase, SymbolicLinkTestCase, + EnvironmentVarGuardTestCase, ) if __name__ == '__main__': diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index b5f28b0eb..cbbd54f0f 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -300,12 +300,14 @@ class NetworkTestCase(unittest.TestCase): def testPythonOrg(self): support.requires('network') - with support.transient_internet('www.python.org'): + with support.transient_internet('www.pythontest.net'): parser = robotparser.RobotFileParser( - "http://www.python.org/robots.txt") + "http://www.pythontest.net/robots.txt") parser.read() self.assertTrue( - parser.can_fetch("*", "http://www.python.org/robots.txt")) + parser.can_fetch("*", "http://www.pythontest.net/")) + self.assertFalse( + parser.can_fetch("*", "http://www.pythontest.net/no-robots-here/")) def load_tests(loader, suite, pattern): suite = unittest.makeSuite(NetworkTestCase) diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py new file mode 100644 index 000000000..1179b7528 --- /dev/null +++ b/Lib/test/test_smtpnet.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import unittest +from test import test_support +import smtplib +import os + +test_support.requires("network") + + +def _external_http_proxy_configured(): + return any(os.environ.get(name) for name in ( + 'http_proxy', 'https_proxy', 'all_proxy', + 'HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY')) + +class SmtpSSLTest(unittest.TestCase): + testServer = 'smtp.gmail.com' + remotePort = 465 + + @unittest.skipIf(_external_http_proxy_configured(), + "external SMTP SSL tests require direct network access") + def test_connect(self): + test_support.get_attribute(smtplib, 'SMTP_SSL') + with test_support.transient_internet(self.testServer): + server = smtplib.SMTP_SSL(self.testServer, self.remotePort) + server.ehlo() + server.quit() + + @unittest.skipIf(_external_http_proxy_configured(), + "external SMTP SSL tests require direct network access") + def test_connect_default_port(self): + test_support.get_attribute(smtplib, 'SMTP_SSL') + with test_support.transient_internet(self.testServer): + server = smtplib.SMTP_SSL(self.testServer) + server.ehlo() + server.quit() + +def test_main(): + test_support.run_unittest(SmtpSSLTest) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 14d707ef8..4ae0b2e67 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -19,6 +19,8 @@ PORT = 50100 HOST = 'localhost' +UNREACHABLE_HOST = java.lang.System.getProperty( + 'jython.test.socket.unreachableHost', '192.0.2.42') MSG = 'Michael Gilfix was here\n' EIGHT_BIT_MSG = 'Bh\xed Al\xe1in \xd3 Cinn\xe9ide anseo\n' os_name = platform.java_ver()[3][0] @@ -221,6 +223,16 @@ def clientSetUp(self): self.cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.cli.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + def clientTearDown(self): + # Do not leave datagram channels for JVM finalization. On Windows, + # especially when regrtest is writing JUnit XML for the whole suite, + # delayed Netty socket cleanup can leave enough old selector/channel + # state around for the following threaded UDP tests to block forever + # waiting for a datagram that was sent by the client side. + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + class SocketConnectedTest(ThreadedTCPSocketTest): def __init__(self, methodName='runTest'): @@ -1358,6 +1370,9 @@ def _testAcceptConnection(self): def testBlockingConnect(self): # Testing blocking connect conn, addr = self.serv.accept() + # Close the accepted server-side socket explicitly. Relying on + # finalization to release the fixed test port can race later tests. + conn.close() def _testBlockingConnect(self): # Testing blocking connect @@ -1480,6 +1495,9 @@ def testCloseFileDoesNotCloseSocket(self): # This test is necessary on java/jython msg = self.cli_conn.recv(1024) self.assertEqual(msg, MSG) + # Close the accepted server-side socket explicitly. Relying on + # finalization to release the fixed test port can race later tests. + self.cli_conn.close() def _testCloseFileDoesNotCloseSocket(self): self.cli_file = self.serv_conn.makefile('wb') @@ -1492,6 +1510,9 @@ def _testCloseFileDoesNotCloseSocket(self): def testCloseSocketDoesNotCloseFile(self): msg = self.cli_conn.recv(1024) self.assertEqual(msg, MSG) + # Close the accepted server-side socket explicitly. Relying on + # finalization to release the fixed test port can race later tests. + self.cli_conn.close() def _testCloseSocketDoesNotCloseFile(self): self.cli_file = self.serv_conn.makefile('wb') @@ -1537,6 +1558,9 @@ def testCloseDoesNotCloseOthers(self): msg = self.cli_conn.recv(len('and ' + MSG)) self.assertEqual(msg, 'and ' + MSG) + # Close the accepted server-side socket explicitly. Relying on + # finalization to release the fixed test port can race later tests. + self.cli_conn.close() def _testCloseDoesNotCloseOthers(self): self.dup_conn1 = self.serv_conn.dup() @@ -1731,7 +1755,7 @@ class TCPClientTimeoutTest(SocketTCPTest): def testConnectTimeout(self): cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) cli.settimeout(0.1) - host = '192.0.2.42' # address in TEST-NET-1, guaranteed to not be routeable + host = UNREACHABLE_HOST try: cli.connect((host, 5000)) except socket.timeout, st: @@ -1747,7 +1771,7 @@ def testConnectDefaultTimeout(self): _saved_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(0.1) cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - host = '192.0.2.42' # address in TEST-NET-1, guaranteed to not be routeable + host = UNREACHABLE_HOST try: cli.connect((host, 5000)) except socket.timeout, st: @@ -2599,6 +2623,13 @@ def setUp(self): self.serverExplicitReady() self.cli_conn, _ = self.serv.accept() + def tearDown(self): + # Close the accepted server-side socket explicitly. Relying on + # finalization to release the fixed test port can race the next test. + self.cli_conn.close() + self.cli_conn = None + SocketTCPTest.tearDown(self) + def clientSetUp(self): self.cli = self.config_client() self.cli.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) diff --git a/Lib/test/test_ssl_jy.py b/Lib/test/test_ssl_jy.py index aad3ba7ca..34decf3b1 100644 --- a/Lib/test/test_ssl_jy.py +++ b/Lib/test/test_ssl_jy.py @@ -3,6 +3,7 @@ # directly in the original, but this reduces the diff and ongoing merge effort import errno +import os import select import socket import ssl @@ -15,6 +16,20 @@ from test.test_ssl import REMOTE_HOST, REMOTE_ROOT_CERT +def _errno_values(*names): + return tuple(getattr(errno, name) for name in names if hasattr(errno, name)) + + +TRANSIENT_CONNECT_EX = _errno_values( + 'ECONNREFUSED', 'EHOSTUNREACH', 'ENETUNREACH', 'ETIMEDOUT', + 'EHOSTDOWN', 'ENETDOWN') + + +def _external_http_proxy_configured(): + return any(os.environ.get(name) for name in ( + 'http_proxy', 'https_proxy', 'all_proxy', + 'HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY')) + class BasicSocketTests(test.test_ssl.BasicSocketTests): @unittest.skip("Jython does not have _ssl, therefore this test needs to be rewritten") @@ -142,7 +157,14 @@ def test_subclass(self): None +@unittest.skipIf(_external_http_proxy_configured(), + "external raw SSL socket tests require direct network access") class NetworkedTests(test.test_ssl.NetworkedTests): + def _skip_if_remote_unreachable(self, rc): + if rc in TRANSIENT_CONNECT_EX: + self.skipTest("%s is not reachable from this environment" % + (REMOTE_HOST,)) + def test_connect_ex(self): # Issue #11326: check connect_ex() implementation with support.transient_internet(REMOTE_HOST): @@ -151,7 +173,9 @@ def test_connect_ex(self): ca_certs=REMOTE_ROOT_CERT) try: # Jython, errno.EISCONN expected per earlier 2.x versions, not 0 - self.assertEqual(errno.EISCONN, s.connect_ex((REMOTE_HOST, 443))) + rc = s.connect_ex((REMOTE_HOST, 443)) + self._skip_if_remote_unreachable(rc) + self.assertEqual(errno.EISCONN, rc) self.assertTrue(s.getpeercert()) finally: s.close() @@ -168,6 +192,7 @@ def test_non_blocking_connect_ex(self): try: s.setblocking(False) rc = s.connect_ex((REMOTE_HOST, 443)) + self._skip_if_remote_unreachable(rc) # EWOULDBLOCK under Windows, EINPROGRESS elsewhere # Jython added EALREADY, as in Jython connect may have already happened self.assertIn(rc, (0, errno.EINPROGRESS, errno.EALREADY, errno.EWOULDBLOCK)) @@ -200,6 +225,7 @@ def test_timeout_connect_ex(self): try: s.settimeout(0.0000001) rc = s.connect_ex((REMOTE_HOST, 443)) + self._skip_if_remote_unreachable(rc) if rc == errno.EISCONN: self.skipTest("REMOTE_HOST responded too quickly") self.assertIn(rc, (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK)) @@ -229,4 +255,3 @@ def test_main(verbose=False): if __name__ == "__main__": test_main() - diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 811d7a778..aeff07ff1 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -872,20 +872,40 @@ class EnvironmentVarGuard(UserDict.DictMixin): def __init__(self): self._environ = os.environ self._changed = {} + self._case_insensitive = getattr(os, '_name', os.name) == 'nt' + + def _changed_key(self, envvar): + if self._case_insensitive: + # Windows environment variable names are case-insensitive even when + # Python code uses different spellings. In Jython this matters for + # tests that deliberately touch both lowercase and uppercase proxy + # variables such as http_proxy and HTTP_PROXY: they are the same + # process environment entry on Windows. If EnvironmentVarGuard + # records those spellings independently, cleanup can restore one + # spelling and then delete it again while "restoring" the other + # spelling that appeared absent after the first unset. Canonicalize + # only the guard's bookkeeping key so all spellings share one saved + # original value, but keep applying changes through the spelling the + # caller supplied. + return envvar.upper() + return envvar + + def _remember_change(self, envvar): + key = self._changed_key(envvar) + if key not in self._changed: + self._changed[key] = envvar, self._environ.get(envvar) def __getitem__(self, envvar): return self._environ[envvar] def __setitem__(self, envvar, value): # Remember the initial value on the first access - if envvar not in self._changed: - self._changed[envvar] = self._environ.get(envvar) + self._remember_change(envvar) self._environ[envvar] = value def __delitem__(self, envvar): # Remember the initial value on the first access - if envvar not in self._changed: - self._changed[envvar] = self._environ.get(envvar) + self._remember_change(envvar) if envvar in self._environ: del self._environ[envvar] @@ -902,12 +922,12 @@ def __enter__(self): return self def __exit__(self, *ignore_exc): - for (k, v) in self._changed.items(): + for (k, (envvar, v)) in self._changed.items(): if v is None: - if k in self._environ: - del self._environ[k] + if envvar in self._environ: + del self._environ[envvar] else: - self._environ[k] = v + self._environ[envvar] = v os.environ = self._environ diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index bd9bb6ff2..e943c8ba4 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1490,14 +1490,19 @@ def test_HTTPError_interface_call(self): def test_main(verbose=None): from test import test_urllib2 - test_support.run_doctest(test_urllib2, verbose) - test_support.run_doctest(urllib2, verbose) - tests = (TrivialTests, - OpenerDirectorTests, - HandlerTests, - MiscTests, - RequestTests) - test_support.run_unittest(*tests) + with test_support.swap_attr(urllib2, '_opener', None): + with test_support.EnvironmentVarGuard() as env: + for name in ('http_proxy', 'https_proxy', 'ftp_proxy', 'no_proxy', + 'HTTP_PROXY', 'HTTPS_PROXY', 'FTP_PROXY', 'NO_PROXY'): + env.unset(name) + test_support.run_doctest(test_urllib2, verbose) + test_support.run_doctest(urllib2, verbose) + tests = (TrivialTests, + OpenerDirectorTests, + HandlerTests, + MiscTests, + RequestTests) + test_support.run_unittest(*tests) if __name__ == "__main__": test_main(verbose=True) diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index 8e7732c8a..b443e0553 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -32,6 +32,17 @@ def wrapped(*args, **kwargs): 'on Travis CI') +def _external_http_proxy_configured(): + return any(os.environ.get(name) for name in ( + 'http_proxy', 'https_proxy', 'all_proxy', + 'HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY')) + + +skip_ftp_test_with_http_proxy = unittest.skipIf( + _external_http_proxy_configured(), + 'external FTP tests require direct FTP access, not an HTTP proxy') + + # Connecting to remote hosts is flaky. Make it more robust by retrying # the connection several times. _urlopen_with_retry = _wrap_with_retry_thrice(urllib2.urlopen, urllib2.URLError) @@ -107,6 +118,7 @@ def setUp(self): # XXX The rest of these tests aren't very good -- they don't check much. # They do sometimes catch some major disasters, though. + @skip_ftp_test_with_http_proxy @skip_ftp_test_on_travis def test_ftp(self): urls = [ @@ -293,6 +305,7 @@ def test_http_timeout(self): FTP_HOST = 'ftp://www.pythontest.net/' + @skip_ftp_test_with_http_proxy @skip_ftp_test_on_travis def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) @@ -300,6 +313,7 @@ def test_ftp_basic(self): u = _urlopen_with_retry(self.FTP_HOST) self.assertIsNone(u.fp.fp._sock.gettimeout()) + @skip_ftp_test_with_http_proxy @skip_ftp_test_on_travis def test_ftp_default_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) @@ -311,6 +325,7 @@ def test_ftp_default_timeout(self): socket.setdefaulttimeout(None) self.assertEqual(u.fp.fp._sock.gettimeout(), 60) + @skip_ftp_test_with_http_proxy @skip_ftp_test_on_travis def test_ftp_no_timeout(self): self.assertIsNone(socket.getdefaulttimeout(),) @@ -322,6 +337,7 @@ def test_ftp_no_timeout(self): socket.setdefaulttimeout(None) self.assertIsNone(u.fp.fp._sock.gettimeout()) + @skip_ftp_test_with_http_proxy @skip_ftp_test_on_travis def test_ftp_timeout(self): with test_support.transient_internet(self.FTP_HOST): diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index bbce16e94..6468cdeb2 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -11,6 +11,9 @@ mimetools = test_support.import_module("mimetools", deprecated=True) +# These urllib tests need a plain HTTP endpoint; www.python.org redirects to HTTPS. +URL = test_support.TEST_HTTP_URL + def _open_with_retry(func, host, *args, **kwargs): # Connecting to remote hosts is flaky. Make it more robust @@ -25,6 +28,12 @@ def _open_with_retry(func, host, *args, **kwargs): raise last_exc +def _external_http_proxy_configured(): + return any(os.environ.get(name) for name in ( + 'http_proxy', 'https_proxy', 'all_proxy', + 'HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY')) + + class URLTimeoutTest(unittest.TestCase): TIMEOUT = 10.0 @@ -36,7 +45,7 @@ def tearDown(self): socket.setdefaulttimeout(None) def testURLread(self): - f = _open_with_retry(urllib.urlopen, "http://www.python.org/") + f = _open_with_retry(urllib.urlopen, URL) x = f.read() class urlopenNetworkTests(unittest.TestCase): @@ -48,7 +57,7 @@ class urlopenNetworkTests(unittest.TestCase): for transparent redirection have been written. setUp is not used for always constructing a connection to - http://www.python.org/ since there a few tests that don't use that address + test_support.TEST_HTTP_URL since there a few tests that don't use that address and making a connection is expensive enough to warrant minimizing unneeded connections. @@ -59,7 +68,7 @@ def urlopen(self, *args): def test_basic(self): # Simple test expected to pass. - open_url = self.urlopen("http://www.python.org/") + open_url = self.urlopen(URL) for attr in ("read", "readline", "readlines", "fileno", "close", "info", "geturl"): self.assertTrue(hasattr(open_url, attr), "object returned from " @@ -71,7 +80,7 @@ def test_basic(self): def test_readlines(self): # Test both readline and readlines. - open_url = self.urlopen("http://www.python.org/") + open_url = self.urlopen(URL) try: self.assertIsInstance(open_url.readline(), basestring, "readline did not return a string") @@ -82,7 +91,7 @@ def test_readlines(self): def test_info(self): # Test 'info'. - open_url = self.urlopen("http://www.python.org/") + open_url = self.urlopen(URL) try: info_obj = open_url.info() finally: @@ -94,12 +103,8 @@ def test_info(self): def test_geturl(self): # Make sure same URL as opened is returned by geturl. - # - # This test has been changed from what's currently in our - # lib-python/2.7 for Jython due to recent updates at the - # python.org to use https; other tests can take advantate of - # URL redirection - URL = "https://www.python.org/" + # Keep this on HTTP: with an HTTP proxy configured, legacy urllib's + # HTTPS proxy path reports geturl() as http://... rather than https://... open_url = self.urlopen(URL) try: gotten_url = open_url.geturl() @@ -109,8 +114,7 @@ def test_geturl(self): def test_getcode(self): # test getcode() with the fancy opener to get 404 error codes - URL = "http://www.python.org/XXXinvalidXXX" - open_url = urllib.FancyURLopener().open(URL) + open_url = urllib.FancyURLopener().open(URL + "/XXXinvalidXXX") try: code = open_url.getcode() finally: @@ -125,7 +129,7 @@ def test_fileno(self): # test can't pass on Windows. return # Make sure fd returned by fileno is valid. - open_url = self.urlopen("http://www.python.org/") + open_url = self.urlopen(URL) fd = open_url.fileno() FILE = os.fdopen(fd) try: @@ -137,6 +141,8 @@ def test_fileno(self): def test_bad_address(self): # Make sure proper exception is raised when connecting to a bogus # address. + if _external_http_proxy_configured(): + self.skipTest("invalid-domain test requires direct DNS lookup") bogus_domain = "sadflkjsasf.i.nvali.d" try: socket.gethostbyname(bogus_domain) @@ -163,7 +169,7 @@ def urlretrieve(self, *args): def test_basic(self): # Test basic functionality. - file_location,info = self.urlretrieve("http://www.python.org/") + file_location,info = self.urlretrieve(URL) self.assertTrue(os.path.exists(file_location), "file location returned by" " urlretrieve is not a valid path") FILE = file(file_location) @@ -176,7 +182,7 @@ def test_basic(self): def test_specified_path(self): # Make sure that specifying the location of the file to write to works. - file_location,info = self.urlretrieve("http://www.python.org/", + file_location,info = self.urlretrieve(URL, test_support.TESTFN) self.assertEqual(file_location, test_support.TESTFN) self.assertTrue(os.path.exists(file_location)) @@ -189,14 +195,15 @@ def test_specified_path(self): def test_header(self): # Make sure header returned as 2nd value from urlretrieve is good. - file_location, header = self.urlretrieve("http://www.python.org/") + file_location, header = self.urlretrieve(URL) os.unlink(file_location) self.assertIsInstance(header, mimetools.Message, "header is not an instance of mimetools.Message") def test_data_header(self): - logo = "http://www.python.org/community/logos/python-logo-master-v3-TM.png" - file_location, fileheaders = self.urlretrieve(logo) + # TEST_HTTP_URL is the pythontest.net root document; urlretrieve stores + # it in a local temporary file, and this test only checks headers. + file_location, fileheaders = self.urlretrieve(URL) os.unlink(file_location) datevalue = fileheaders.getheader('Date') dateformat = '%a, %d %b %Y %H:%M:%S GMT' diff --git a/extlibs/profile.properties b/extlibs/profile.properties index 0acd965a9..1dcb93e20 100644 --- a/extlibs/profile.properties +++ b/extlibs/profile.properties @@ -52,7 +52,8 @@ file=profile.txt # (comma separated) # Note: com.mentorgen.tools.profile is always excluded # -include=python +include=org.python +exclude=org.python.core.util.LimitedCache # # Track Object Allocation (very expensive) # values: on, off @@ -66,4 +67,6 @@ output=text #clock-resolution=ms output-summary-only=yes -accept-class-loaders=org.python.core.BytecodeLoader +# Use the default application class loader. Limiting this to BytecodeLoader +# prevents simple command-line scripts from producing profile output. +#accept-class-loaders=org.python.core.BytecodeLoader diff --git a/lib-python/2.7/test/test_urllib2net.py b/lib-python/2.7/test/test_urllib2net.py index 4d73ac0ac..1cd80f283 100644 --- a/lib-python/2.7/test/test_urllib2net.py +++ b/lib-python/2.7/test/test_urllib2net.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import unittest from test import test_support from test.test_urllib2 import sanepathname2url @@ -27,6 +25,13 @@ def wrapped(*args, **kwargs): return _retry_thrice(func, exc, *args, **kwargs) return wrapped +# bpo-35411: FTP tests of test_urllib2net randomly fail +# with "425 Security: Bad IP connecting" on Travis CI +skip_ftp_test_on_travis = unittest.skipIf('TRAVIS' in os.environ, + 'bpo-35411: skip FTP test ' + 'on Travis CI') + + # Connecting to remote hosts is flaky. Make it more robust by retrying # the connection several times. _urlopen_with_retry = _wrap_with_retry_thrice(urllib2.urlopen, urllib2.URLError) @@ -80,13 +85,13 @@ def test_close(self): # underlying socket # delve deep into response to fetch socket._socketobject - response = _urlopen_with_retry("http://www.python.org/") + response = _urlopen_with_retry(test_support.TEST_HTTP_URL) abused_fileobject = response.fp - self.assertTrue(abused_fileobject.__class__ is socket._fileobject) + self.assertIs(abused_fileobject.__class__, socket._fileobject) httpresponse = abused_fileobject._sock - self.assertTrue(httpresponse.__class__ is httplib.HTTPResponse) + self.assertIs(httpresponse.__class__, httplib.HTTPResponse) fileobject = httpresponse.fp - self.assertTrue(fileobject.__class__ is socket._fileobject) + self.assertIs(fileobject.__class__, socket._fileobject) self.assertTrue(not fileobject.closed) response.close() @@ -102,13 +107,12 @@ def setUp(self): # XXX The rest of these tests aren't very good -- they don't check much. # They do sometimes catch some major disasters, though. + @skip_ftp_test_on_travis def test_ftp(self): urls = [ - 'ftp://ftp.kernel.org/pub/linux/kernel/README', - 'ftp://ftp.kernel.org/pub/linux/kernel/non-existent-file', - #'ftp://ftp.kernel.org/pub/leenox/kernel/test', - 'ftp://gatekeeper.research.compaq.com/pub/DEC/SRC' - '/research-reports/00README-Legal-Rules-Regs', + 'ftp://www.pythontest.net/README', + ('ftp://www.pythontest.net/non-existent-file', + None, urllib2.URLError), ] self._test_urls(urls, self._extra_handlers()) @@ -157,15 +161,15 @@ def test_file(self): ## self._test_urls(urls, self._extra_handlers()+[bauth, dauth]) def test_urlwithfrag(self): - urlwith_frag = "http://docs.python.org/2/glossary.html#glossary" + urlwith_frag = "http://www.pythontest.net/index.html#frag" with test_support.transient_internet(urlwith_frag): req = urllib2.Request(urlwith_frag) res = urllib2.urlopen(req) self.assertEqual(res.geturl(), - "http://docs.python.org/2/glossary.html#glossary") + "http://www.pythontest.net/index.html#frag") def test_fileno(self): - req = urllib2.Request("http://www.python.org") + req = urllib2.Request(test_support.TEST_HTTP_URL) opener = urllib2.build_opener() res = opener.open(req) try: @@ -176,7 +180,7 @@ def test_fileno(self): res.close() def test_custom_headers(self): - url = "http://www.example.com" + url = test_support.TEST_HTTP_URL with test_support.transient_internet(url): opener = urllib2.build_opener() request = urllib2.Request(url) @@ -188,6 +192,7 @@ def test_custom_headers(self): opener.open(request) self.assertEqual(request.get_header('User-agent'),'Test-Agent') + @unittest.skip('XXX: http://www.imdb.com is gone') def test_sites_no_connection_close(self): # Some sites do not send Connection: close header. # Verify that those work properly. (#issue12576) @@ -252,15 +257,15 @@ def _extra_handlers(self): class TimeoutTest(unittest.TestCase): def test_http_basic(self): - self.assertTrue(socket.getdefaulttimeout() is None) - url = "http://www.python.org" + self.assertIsNone(socket.getdefaulttimeout()) + url = test_support.TEST_HTTP_URL with test_support.transient_internet(url, timeout=None): u = _urlopen_with_retry(url) - self.assertTrue(u.fp._sock.fp._sock.gettimeout() is None) + self.assertIsNone(u.fp._sock.fp._sock.gettimeout()) def test_http_default_timeout(self): - self.assertTrue(socket.getdefaulttimeout() is None) - url = "http://www.python.org" + self.assertIsNone(socket.getdefaulttimeout()) + url = test_support.TEST_HTTP_URL with test_support.transient_internet(url): socket.setdefaulttimeout(60) try: @@ -270,32 +275,34 @@ def test_http_default_timeout(self): self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 60) def test_http_no_timeout(self): - self.assertTrue(socket.getdefaulttimeout() is None) - url = "http://www.python.org" + self.assertIsNone(socket.getdefaulttimeout()) + url = test_support.TEST_HTTP_URL with test_support.transient_internet(url): socket.setdefaulttimeout(60) try: u = _urlopen_with_retry(url, timeout=None) finally: socket.setdefaulttimeout(None) - self.assertTrue(u.fp._sock.fp._sock.gettimeout() is None) + self.assertIsNone(u.fp._sock.fp._sock.gettimeout()) def test_http_timeout(self): - url = "http://www.python.org" + url = test_support.TEST_HTTP_URL with test_support.transient_internet(url): u = _urlopen_with_retry(url, timeout=120) self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120) - FTP_HOST = "ftp://ftp.mirror.nl/pub/gnu/" + FTP_HOST = 'ftp://www.pythontest.net/' + @skip_ftp_test_on_travis def test_ftp_basic(self): - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) with test_support.transient_internet(self.FTP_HOST, timeout=None): u = _urlopen_with_retry(self.FTP_HOST) - self.assertTrue(u.fp.fp._sock.gettimeout() is None) + self.assertIsNone(u.fp.fp._sock.gettimeout()) + @skip_ftp_test_on_travis def test_ftp_default_timeout(self): - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout()) with test_support.transient_internet(self.FTP_HOST): socket.setdefaulttimeout(60) try: @@ -304,16 +311,18 @@ def test_ftp_default_timeout(self): socket.setdefaulttimeout(None) self.assertEqual(u.fp.fp._sock.gettimeout(), 60) + @skip_ftp_test_on_travis def test_ftp_no_timeout(self): - self.assertTrue(socket.getdefaulttimeout() is None) + self.assertIsNone(socket.getdefaulttimeout(),) with test_support.transient_internet(self.FTP_HOST): socket.setdefaulttimeout(60) try: u = _urlopen_with_retry(self.FTP_HOST, timeout=None) finally: socket.setdefaulttimeout(None) - self.assertTrue(u.fp.fp._sock.gettimeout() is None) + self.assertIsNone(u.fp.fp._sock.gettimeout()) + @skip_ftp_test_on_travis def test_ftp_timeout(self): with test_support.transient_internet(self.FTP_HOST): u = _urlopen_with_retry(self.FTP_HOST, timeout=60) diff --git a/src/shell/jython b/src/shell/jython index 65b39cb92..558dd2421 100755 --- a/src/shell/jython +++ b/src/shell/jython @@ -13,6 +13,7 @@ # ----------------------------------------------------------------------------- cygwin=false +jdb_requested=false # ----- Identify OS we are running under -------------------------------------- case "`uname`" in @@ -73,25 +74,30 @@ if [ -z "$JYTHON_OPTS" ] ; then fi CP_DELIMITER=":" +EXTRA_CP="" -CP=$JYTHON_HOME/jython-dev.jar - -if [ -f "$CP" ] ; then - # We are running in the development environment. - # Add -test JAR to class path. - CP="$CP$CP_DELIMITER$JYTHON_HOME/jython-test.jar" - # Add necessary jar dir for command-line execution - CP="$CP$CP_DELIMITER$JYTHON_HOME/javalib/*" -elif [ ! -f "$JYTHON_HOME"/jython.jar ] ; then - echo "$0: $JYTHON_HOME contains neither jython-dev.jar nor jython.jar." >&2 - echo "Try running this script from the 'bin' directory of an installed Jython or " >&2 - echo 'setting $JYTHON_HOME.' >&2 - exit 1 -else - # We are running in the deployment environment. - # Class path is just the main JAR (not tests). - CP=$JYTHON_HOME/jython.jar -fi +build_jython_classpath() { + CP=$1/jython-dev.jar + + if [ -f "$CP" ] ; then + # We are running in the development environment. + # Add -test JAR to class path. + CP="$CP$CP_DELIMITER$1/jython-test.jar" + # Add necessary jar dir for command-line execution + CP="$CP$CP_DELIMITER$1/javalib/*" + elif [ ! -f "$1"/jython.jar ] ; then + echo "$0: $1 contains neither jython-dev.jar nor jython.jar." >&2 + echo "Try running this script from the 'bin' directory of an installed Jython or " >&2 + echo 'setting $JYTHON_HOME.' >&2 + exit 1 + else + # We are running in the deployment environment. + # Class path is just the main JAR (not tests). + CP=$1/jython.jar + fi +} + +build_jython_classpath "$JYTHON_HOME" if $cygwin; then # switch delimiter only after building a Unix style $CP @@ -101,6 +107,10 @@ if $cygwin; then PRG=`cygpath -w "$PRG"`.exe fi +JYTHON_HOME_FOR_JAVA="$JYTHON_HOME" +PRG_FOR_JAVA="$PRG" +JDB_TMPDIR="" + # ----- Execute the requested command ----------------------------------------- if [ -z "$JAVA_MEM" ] ; then @@ -134,10 +144,10 @@ while [ $# -gt 0 ] ; do echo "(Prepend -J in front of these options when using 'jython' command)" exit elif [ "${val}" = "-classpath" ]; then - CP="$CP$CP_DELIMITER$2" + EXTRA_CP="$EXTRA_CP$CP_DELIMITER$2" shift elif [ "${val}" = "-cp" ]; then - CP="$CP$CP_DELIMITER$2" + EXTRA_CP="$EXTRA_CP$CP_DELIMITER$2" shift else if [ "${val:0:16}" = "-Dfile.encoding=" ]; then @@ -182,6 +192,7 @@ while [ $# -gt 0 ] ; do ;; # Run under JDB --jdb) + jdb_requested=true if [ -z "$JAVA_HOME" ] ; then JAVA_CMD=(jdb) else @@ -222,8 +233,100 @@ python_args=("${python_args[@]}" "$@") # Put the python_args back into the position arguments $1, $2 etc set -- "${python_args[@]}" +setup_jdb_path_alias() { + if ! $jdb_requested || $cygwin; then + return + fi + + # Some jdb implementations, notably OpenJDK/Temurin 8, split the debuggee + # command line on spaces after receiving it from this launcher. Use a + # temporary no-space symlink for Jython paths so the classpath and + # -Dpython.* values survive jdb's second-stage JVM launch. + case "$JYTHON_HOME$PRG" in + *" "*) + JDB_TMPDIR=`mktemp -dt jython-jdb.XXXXXX` || exit 1 + JDB_HOME_TARGET=`cd "$JYTHON_HOME" && pwd` || exit 1 + ln -s "$JDB_HOME_TARGET" "$JDB_TMPDIR/jython" || exit 1 + JYTHON_HOME_FOR_JAVA="$JDB_TMPDIR/jython" + build_jython_classpath "$JYTHON_HOME_FOR_JAVA" + + PRG_FOR_JAVA="$JYTHON_HOME_FOR_JAVA/bin/`basename "$PRG"`" + if [ ! -e "$PRG_FOR_JAVA" ]; then + mkdir "$JDB_TMPDIR/bin" || exit 1 + case "$PRG" in + /*) JDB_PRG_TARGET="$PRG" ;; + *) JDB_PRG_TARGET="`pwd`/$PRG" ;; + esac + ln -s "$JDB_PRG_TARGET" "$JDB_TMPDIR/bin/`basename "$PRG"`" || exit 1 + PRG_FOR_JAVA="$JDB_TMPDIR/bin/`basename "$PRG"`" + fi + trap 'rm -rf "$JDB_TMPDIR"' EXIT + ;; + esac +} + +java_major_version() { + local output version major + output=$("${JAVA_CMD[@]}" -version 2>&1) + version=`expr "$output" : '.*version "\([^"]*\)"'` + case "$version" in + 1.*) + major=${version#1.} + major=${major%%.*} + ;; + *) + major=${version%%[!0-9]*} + ;; + esac + echo "$major" +} + +append_java_arg() { + if $jdb_requested; then + java_args=("${java_args[@]}" "-R$1") + else + java_args=("${java_args[@]}" "$1") + fi +} + +if [ -z "$help_requested" ]; then + java_major=`java_major_version` + if [ -n "$java_major" ]; then + if [ "$java_major" -ge 9 ] 2>/dev/null; then + case " $JAVA_OPTS ${java_args[*]} " in + *--add-opens=java.base/java.io*) ;; + *) append_java_arg --add-opens=java.base/java.io=ALL-UNNAMED ;; + esac + case " $JAVA_OPTS ${java_args[*]} " in + *--add-opens=java.base/sun.nio.ch*) ;; + *) append_java_arg --add-opens=java.base/sun.nio.ch=ALL-UNNAMED ;; + esac + fi + if [ "$java_major" -ge 25 ] 2>/dev/null; then + case " $JAVA_OPTS ${java_args[*]} " in + *--enable-native-access*) ;; + *) append_java_arg --enable-native-access=ALL-UNNAMED ;; + esac + case " $JAVA_OPTS ${java_args[*]} " in + *--sun-misc-unsafe-memory-access*) ;; + *) append_java_arg --sun-misc-unsafe-memory-access=allow ;; + esac + fi + fi +fi + JAVA_OPTS="$JAVA_OPTS $JAVA_MEM $JAVA_STACK" +if [ -n "$profile_requested" ]; then + java_major=`java_major_version` + if [ -n "$java_major" ] && [ "$java_major" -ge 13 ] 2>/dev/null; then + echo "--profile is not supported on Java $java_major; the bundled Java Interactive Profiler is only compatible with Java 12 and earlier." >&2 + exit 2 + fi +fi + +setup_jdb_path_alias + if $cygwin; then JAVA_HOME=`cygpath --mixed "$JAVA_HOME"` JYTHON_HOME=`cygpath --mixed "$JYTHON_HOME"` @@ -244,12 +347,18 @@ fi if [ -n "$profile_requested" -o -z "$boot_requested" ] ; then [ -n "$profile_requested" ] && echo "Running with instrumented profiler" - java_args=("${java_args[@]}" -classpath "$CP$CP_DELIMITER$CLASSPATH") + if [ -n "$profile_requested" ]; then + java_major=`java_major_version` + if [ -n "$java_major" ] && [ "$java_major" -lt 13 ] 2>/dev/null; then + java_args=("${java_args[@]}" -Xverify:none) + fi + fi + java_args=("${java_args[@]}" -classpath "$CP$EXTRA_CP$CP_DELIMITER$CLASSPATH") else - if [ -z "$help_requested" -a -z "$print_requested" ] ; then + if [ -z "$help_requested" -a -z "$print_requested" ] && ! $jdb_requested ; then JAVA_CMD=(exec "${JAVA_CMD[@]}") fi - java_args=("${java_args[@]}" -Xbootclasspath/a:"$CP") + java_args=("${java_args[@]}" -Xbootclasspath/a:"$CP$EXTRA_CP") [ -n "$CLASSPATH" ] && java_args=("${java_args[@]}" -classpath "$CLASSPATH") fi @@ -257,14 +366,15 @@ if [ -n "$print_requested" ] ; then JAVA_CMD=(echo $JAVA_CMD) fi -"${JAVA_CMD[@]}" $JAVA_OPTS "${java_args[@]}" -Dpython.home="$JYTHON_HOME" \ - -Dpython.executable="$PRG" org.python.util.jython $JYTHON_OPTS "$@" +"${JAVA_CMD[@]}" $JAVA_OPTS "${java_args[@]}" -Dpython.home="$JYTHON_HOME_FOR_JAVA" \ + -Dpython.executable="$PRG_FOR_JAVA" org.python.util.jython $JYTHON_OPTS "$@" JYTHON_STATUS=$? if [ -n "$profile_requested" ] ; then echo "Profiling results:" cat profile.txt elif [ -n "$help_requested" ] ; then + echo "" >&2 echo "Jython launcher options:" >&2 echo "-Jarg : pass argument through to Java VM (e.g. -J-Xmx512m)" >&2 echo "--jdb : run under JDB" >&2 diff --git a/src/shell/jython.exe b/src/shell/jython.exe index bdaa44712..9dceb807e 100755 Binary files a/src/shell/jython.exe and b/src/shell/jython.exe differ diff --git a/src/shell/jython.py b/src/shell/jython.py index fa43c0eaa..7eb1386c6 100644 --- a/src/shell/jython.py +++ b/src/shell/jython.py @@ -15,9 +15,12 @@ import glob import os import os.path +import re +import shutil import shlex import subprocess import sys +import tempfile from collections import OrderedDict @@ -160,6 +163,7 @@ class JythonCommand(object): def __init__(self, args, jython_args): self.args = args self.jython_args = jython_args + self._jdb_alias_dir = None @property def uname(self): @@ -247,6 +251,65 @@ def jython_home(self): self._jython_home = home return self._jython_home + def needs_jdb_path_alias(self): + """Return whether jdb needs a no-space alias for Jython paths. + + Some jdb implementations, notably OpenJDK/Temurin 8, split the + debuggee command line on spaces after receiving it from this launcher. + A temporary symlink without spaces lets the classpath and -Dpython.* + values survive jdb's second-stage JVM launch. + """ + return (self.args.jdb and self.uname not in (u"windows", u"cygwin") + and (u" " in self.jython_home or u" " in self.executable)) + + def ensure_jdb_alias_dir(self): + if self._jdb_alias_dir is None: + alias_dir = tempfile.mkdtemp(prefix="jython-jdb.") + if isinstance(alias_dir, bytes): + alias_dir = alias_dir.decode(ENCODING) + self._jdb_alias_dir = alias_dir + return self._jdb_alias_dir + + @property + def jython_home_for_java(self): + if hasattr(self, "_jython_home_for_java"): + return self._jython_home_for_java + if self.needs_jdb_path_alias(): + alias = os.path.join(self.ensure_jdb_alias_dir(), u"jython") + if not os.path.exists(alias): + os.symlink(os.path.abspath(self.jython_home), alias) + self._jython_home_for_java = alias + else: + self._jython_home_for_java = self.jython_home + return self._jython_home_for_java + + @property + def executable_for_java(self): + if hasattr(self, "_executable_for_java"): + return self._executable_for_java + if self.needs_jdb_path_alias(): + alias = os.path.join(self.jython_home_for_java, u"bin", + os.path.basename(self.executable)) + if not os.path.exists(alias) and u" " in self.executable: + bin_dir = os.path.join(self.ensure_jdb_alias_dir(), u"bin") + if not os.path.exists(bin_dir): + os.mkdir(bin_dir) + alias = os.path.join(bin_dir, os.path.basename(self.executable)) + if not os.path.exists(alias): + os.symlink(os.path.abspath(self.executable), alias) + if os.path.exists(alias): + self._executable_for_java = alias + else: + self._executable_for_java = self.executable + else: + self._executable_for_java = self.executable + return self._executable_for_java + + def cleanup(self): + if self._jdb_alias_dir is not None: + shutil.rmtree(self._jdb_alias_dir, ignore_errors=True) + self._jdb_alias_dir = None + @property def jython_opts(): return get_env("JYTHON_OPTS", "") @@ -260,7 +323,7 @@ def jython_jars(self): if hasattr(self, "_jython_jars"): return self._jython_jars - home = self.jython_home + home = self.jython_home_for_java jython_jar = os.path.join(home, 'jython.jar') jython_dev_jar = os.path.join(home, 'jython-dev.jar') @@ -308,9 +371,32 @@ def java_stack(self): def java_opts(self): return [self.java_mem, self.java_stack] + @property + def java_major_version(self): + if hasattr(self, "_java_major_version"): + return self._java_major_version + try: + output = subprocess.check_output([self.java_command, u"-version"], + stderr=subprocess.STDOUT) + if isinstance(output, bytes): + output = output.decode(ENCODING, "replace") + match = re.search(r'version "([^"]+)"', output) + if match is None: + major = 0 + else: + version = match.group(1) + if version.startswith(u"1."): + major = int(version.split(u".", 2)[1]) + else: + major = int(re.match(r"\d+", version).group(0)) + except Exception: + major = 0 + self._java_major_version = major + return major + @property def java_profile_agent(self): - return os.path.join(self.jython_home, "javalib", "profile.jar") + return os.path.join(self.jython_home_for_java, "javalib", "profile.jar") def set_encoding(self): if "JAVA_ENCODING" not in os.environ and self.uname == "darwin" and "file.encoding" not in self.args.properties: @@ -346,11 +432,36 @@ def command(self): # Set default file encoding just for Darwin (?) self.set_encoding() + if self.args.profile and self.java_major_version >= 13: + msg = (u"--profile is not supported on Java {0}; the bundled Java " + u"Interactive Profiler is only compatible with Java 12 and earlier.") + print >> sys.stderr, msg.format(self.java_major_version) + sys.exit(2) + # Begin to build the Java part of the ultimate command args = [self.java_command] args.extend(self.java_opts) args.extend(self.args.java) + def append_java_arg(arg): + if self.args.jdb: + args.append(u"-R%s" % arg) + else: + args.append(arg) + + if (not self.args.help and self.java_major_version >= 9 + and not any(a.startswith(u"--add-opens=java.base/java.io") for a in args)): + append_java_arg(u"--add-opens=java.base/java.io=ALL-UNNAMED") + if (not self.args.help and self.java_major_version >= 9 + and not any(a.startswith(u"--add-opens=java.base/sun.nio.ch") for a in args)): + append_java_arg(u"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED") + if (not self.args.help and self.java_major_version >= 25 + and not any(a.startswith(u"--enable-native-access") for a in args)): + append_java_arg(u"--enable-native-access=ALL-UNNAMED") + if (not self.args.help and self.java_major_version >= 25 + and not any(a.startswith(u"--sun-misc-unsafe-memory-access") for a in args)): + append_java_arg(u"--sun-misc-unsafe-memory-access=allow") + # Get the class path right (depends on --boot) classpath = self.java_classpath jython_jars = self.jython_jars @@ -361,9 +472,9 @@ def command(self): args.extend([u"-classpath", self.convert_path(classpath)]) if "python.home" not in self.args.properties: - args.append(u"-Dpython.home=%s" % self.convert_path(self.jython_home)) + args.append(u"-Dpython.home=%s" % self.convert_path(self.jython_home_for_java)) if "python.executable" not in self.args.properties: - args.append(u"-Dpython.executable=%s" % self.convert_path(self.executable)) + args.append(u"-Dpython.executable=%s" % self.convert_path(self.executable_for_java)) if "python.launcher.uname" not in self.args.properties: args.append(u"-Dpython.launcher.uname=%s" % self.uname) @@ -377,6 +488,8 @@ def command(self): args.append(u"-Dpython.console=org.python.core.PlainConsole") if self.args.profile: + if self.java_major_version < 13: + args.append(u"-Xverify:none") args.append(u"-XX:-UseSplitVerifier") args.append(u"-javaagent:%s" % self.convert_path(self.java_profile_agent)) @@ -571,10 +684,11 @@ def main(sys_args): # It is possible the Unicode cannot be encoded for the console enc = sys.stdout.encoding or 'ascii' sys.stdout.write(command_line.encode(enc, 'replace') + "\n") + jython_command.cleanup() else: try: if not (is_windows or not hasattr(os, "execvp") or args.help or - jython_command.uname == u"cygwin"): + jython_command.uname == u"cygwin" or args.jdb): # Replace this process with the java process. # # NB such replacements actually do not work under Windows, @@ -586,9 +700,12 @@ def main(sys_args): else: result = 1 try: - result = subprocess.call(encode_list(command)) - if args.help: - print_help() + try: + result = subprocess.call(encode_list(command)) + if args.help: + print_help() + finally: + jython_command.cleanup() except KeyboardInterrupt: pass sys.exit(result)