From 0141ffaefddbbaf7bf35b2b8ac3fdeab7b8cf325 Mon Sep 17 00:00:00 2001 From: Roger Serwy Date: Wed, 3 Apr 2013 00:42:24 -0500 Subject: [PATCH 001/480] #14254: IDLE now handles readline correctly across shell restarts. --- Lib/idlelib/PyShell.py | 14 ++++++++++++-- Misc/NEWS | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/PyShell.py b/Lib/idlelib/PyShell.py index d144a51b725..9451d2e6c92 100644 --- a/Lib/idlelib/PyShell.py +++ b/Lib/idlelib/PyShell.py @@ -469,6 +469,7 @@ def restart_subprocess(self, with_cwd=False): self.display_no_subprocess_error() return None self.transfer_path(with_cwd=with_cwd) + console.stop_readline() # annotate restart in shell window and mark it console.text.delete("iomark", "end-1c") if was_executing: @@ -908,6 +909,7 @@ def get_standard_extension_names(self): canceled = False endoffile = False closing = False + _stop_readline_flag = False def set_warning_stream(self, stream): global warning_stream @@ -983,8 +985,7 @@ def close(self): parent=self.text) if response is False: return "cancel" - if self.reading: - self.top.quit() + self.stop_readline() self.canceled = True self.closing = True # Wait for poll_subprocess() rescheduling to stop @@ -1036,6 +1037,12 @@ def begin(self): Tkinter._default_root = None # 03Jan04 KBK What's this? return True + def stop_readline(self): + if not self.reading: # no nested mainloop to exit. + return + self._stop_readline_flag = True + self.top.quit() + def readline(self): save = self.reading try: @@ -1043,6 +1050,9 @@ def readline(self): self.top.mainloop() # nested mainloop() finally: self.reading = save + if self._stop_readline_flag: + self._stop_readline_flag = False + return "" line = self.text.get("iomark", "end-1c") if len(line) == 0: # may be EOF if we quit our mainloop with Ctrl-C line = "\n" diff --git a/Misc/NEWS b/Misc/NEWS index 75141c61062..b5dcdeb96b4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,8 @@ Core and Builtins Library ------- +- Issue #14254: IDLE now handles readline correctly across shell restarts. + - Issue #17614: IDLE no longer raises exception when quickly closing a file. - Issue #13163: Rename operands in smtplib.SMTP._get_socket to correct names; From 506fe321434f0d681a984350c20b5824af657baa Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Wed, 3 Apr 2013 12:34:57 -0400 Subject: [PATCH 002/480] Issue #15940: Specify effect of locale on time functions. --- Doc/library/time.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 4ff0ce473e6..e13be609bfb 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -547,8 +547,8 @@ The module defines the following functions and data items: More object-oriented interface to dates and times. Module :mod:`locale` - Internationalization services. The locale settings can affect the return values - for some of the functions in the :mod:`time` module. + Internationalization services. The locale setting affects the interpretation + of many format specifiers in :func:`strftime` and :func:`strptime`. Module :mod:`calendar` General calendar-related functions. :func:`timegm` is the inverse of From 3db774c95af9eaa0cf4f670017809bc4b2f236bd Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Wed, 3 Apr 2013 12:45:24 -0400 Subject: [PATCH 003/480] Issue #15940: Replace tab. --- Doc/library/time.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index e13be609bfb..a4de69f70aa 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -548,7 +548,7 @@ The module defines the following functions and data items: Module :mod:`locale` Internationalization services. The locale setting affects the interpretation - of many format specifiers in :func:`strftime` and :func:`strptime`. + of many format specifiers in :func:`strftime` and :func:`strptime`. Module :mod:`calendar` General calendar-related functions. :func:`timegm` is the inverse of From 280b9e1a81b7ccb077a408cf47d51054a11fa3db Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Wed, 3 Apr 2013 13:24:26 -0400 Subject: [PATCH 004/480] Issue #15940: NEWS entry --- Misc/NEWS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index b5dcdeb96b4..2ee15f222fc 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,11 @@ Library - Issue #17526: fix an IndexError raised while passing code without filename to inspect.findsource(). Initial patch by Tyler Doyle. +Documentation +------------- + +- Issue #15940: Specify effect of locale on time functions. + What's New in Python 2.7.4? =========================== From f6d515fc600198d5006d5af60691d20985e4dd16 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 3 Apr 2013 22:35:12 -0400 Subject: [PATCH 005/480] close search and replace dialog after it is used (closes #17625) --- Lib/idlelib/ReplaceDialog.py | 1 + Misc/NEWS | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Lib/idlelib/ReplaceDialog.py b/Lib/idlelib/ReplaceDialog.py index ae440353f7b..54c270df23c 100644 --- a/Lib/idlelib/ReplaceDialog.py +++ b/Lib/idlelib/ReplaceDialog.py @@ -122,6 +122,7 @@ def replace_all(self, event=None): text.undo_block_stop() if first and last: self.show_hit(first, last) + self.close() def do_find(self, ok=0): if not self.engine.getprog(): diff --git a/Misc/NEWS b/Misc/NEWS index 45873092c37..6f4740c80bb 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,8 @@ Core and Builtins Library ------- +- Issue #17625: In IDLE, close the replace dialog after it is used. + - Issue #17531: Fix tests that thought group and user ids were always the int type. Also, always allow -1 as a valid group and user id. From 989fc384c69fb631adf8c847b16f5cb26014c974 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Thu, 4 Apr 2013 09:16:15 +0300 Subject: [PATCH 006/480] #8913: add examples and docs for date/time/datetime.__format__. Patch by Heikki Partanen. --- Doc/library/datetime.rst | 27 +++++++++++++++++++++++++++ Misc/ACKS | 1 + 2 files changed, 28 insertions(+) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 409fd8bead2..7219cea6457 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -555,6 +555,13 @@ Instance methods: section :ref:`strftime-strptime-behavior`. +.. method:: date.__format__(format) + + Same as :meth:`.date.strftime`. This makes it possible to specify format + string for a :class:`.date` object when using :meth:`str.format`. + See section :ref:`strftime-strptime-behavior`. + + Example of counting days to an event:: >>> import time @@ -605,6 +612,8 @@ Example of working with :class:`date`: '11/03/02' >>> d.strftime("%A %d. %B %Y") 'Monday 11. March 2002' + >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") + 'The day is 11, the month is March.' .. _datetime-datetime: @@ -1044,6 +1053,13 @@ Instance methods: string. See section :ref:`strftime-strptime-behavior`. +.. method:: datetime.__format__(format) + + Same as :meth:`.datetime.strftime`. This makes it possible to specify format + string for a :class:`.datetime` object when using :meth:`str.format`. + See section :ref:`strftime-strptime-behavior`. + + Examples of working with datetime objects: .. doctest:: @@ -1088,6 +1104,8 @@ Examples of working with datetime objects: >>> # Formatting datetime >>> dt.strftime("%A, %d. %B %Y %I:%M%p") 'Tuesday, 21. November 2006 04:30PM' + >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time") + 'The day is 21, the month is November, the time is 04:30PM.' Using datetime with tzinfo: @@ -1268,6 +1286,13 @@ Instance methods: See section :ref:`strftime-strptime-behavior`. +.. method:: time.__format__(format) + + Same as :meth:`.time.strftime`. This makes it possible to specify format string + for a :class:`.time` object when using :meth:`str.format`. + See section :ref:`strftime-strptime-behavior`. + + .. method:: time.utcoffset() If :attr:`tzinfo` is ``None``, returns ``None``, else returns @@ -1314,6 +1339,8 @@ Example: 'Europe/Prague' >>> t.strftime("%H:%M:%S %Z") '12:10:30 Europe/Prague' + >>> 'The {} is {:%H:%M}.'.format("time", t) + 'The time is 12:10.' .. _datetime-tzinfo: diff --git a/Misc/ACKS b/Misc/ACKS index 2d6bd07892f..a91dc327e94 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -761,6 +761,7 @@ Peter Parente Alexandre Parenteau Dan Parisien William Park +Heikki Partanen Harri Pasanen Randy Pausch Samuele Pedroni From 1567901e3679e6c7e35e5ac83d50cbf8c9160114 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 5 Apr 2013 10:10:27 +0300 Subject: [PATCH 007/480] Fix typo. --- Doc/howto/argparse.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 449f7cd96f0..1e608eca18e 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -11,7 +11,7 @@ recommended command-line parsing module in the Python standard library. .. note:: - There's two other modules that fulfill the same task, namely + There are two other modules that fulfill the same task, namely :mod:`getopt` (an equivalent for :c:func:`getopt` from the C language) and the deprecated :mod:`optparse`. Note also that :mod:`argparse` is based on :mod:`optparse`, From 77ea9d9bf8162f2ae54a936bbead22d5db93c444 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 5 Apr 2013 11:39:36 +0300 Subject: [PATCH 008/480] Fix typo --- Doc/howto/argparse.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index c0b3f94da0e..3d49d359292 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -668,7 +668,7 @@ Conflicting options So far, we have been working with two methods of an :class:`argparse.ArgumentParser` instance. Let's introduce a third one, :meth:`add_mutually_exclusive_group`. It allows for us to specify options that -conflict with each other. Let's also change the rest of the program make the +conflict with each other. Let's also change the rest of the program to make the new functionality makes more sense: we'll introduce the ``--quiet`` option, which will be the opposite of the ``--verbose`` one:: From a4b09b925db4405ad9ff0486ed68e25cc0399783 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 5 Apr 2013 16:21:39 +0300 Subject: [PATCH 009/480] Add link to glossary for hashable term in docs for set type --- Doc/library/stdtypes.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 2761410f0a1..5d6d35343e0 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1738,9 +1738,10 @@ The constructors for both classes work the same: frozenset([iterable]) Return a new set or frozenset object whose elements are taken from - *iterable*. The elements of a set must be hashable. To represent sets of - sets, the inner sets must be :class:`frozenset` objects. If *iterable* is - not specified, a new empty set is returned. + *iterable*. The elements of a set must be :term:`hashable`. To + represent sets of sets, the inner sets must be :class:`frozenset` + objects. If *iterable* is not specified, a new empty set is + returned. Instances of :class:`set` and :class:`frozenset` provide the following operations: From cb74314956f72c29fc37826f545ea3b98822b48f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola' Date: Sat, 6 Apr 2013 03:46:47 +0200 Subject: [PATCH 010/480] #17641: 2.X / 3.X ssl doc unification --- Doc/library/ssl.rst | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 19b90a15cb7..1ddc0f7b423 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -298,14 +298,30 @@ Functions, Constants, and Exceptions SSLSocket Objects ----------------- -.. method:: SSLSocket.read([nbytes=1024]) - - Reads up to ``nbytes`` bytes from the SSL-encrypted channel and returns them. - -.. method:: SSLSocket.write(data) - - Writes the ``data`` to the other side of the connection, using the SSL - channel to encrypt. Returns the number of bytes written. +SSL sockets provide the following methods of :ref:`socket-objects`: + +- :meth:`~socket.socket.accept()` +- :meth:`~socket.socket.bind()` +- :meth:`~socket.socket.close()` +- :meth:`~socket.socket.connect()` +- :meth:`~socket.socket.fileno()` +- :meth:`~socket.socket.getpeername()`, :meth:`~socket.socket.getsockname()` +- :meth:`~socket.socket.getsockopt()`, :meth:`~socket.socket.setsockopt()` +- :meth:`~socket.socket.gettimeout()`, :meth:`~socket.socket.settimeout()`, + :meth:`~socket.socket.setblocking()` +- :meth:`~socket.socket.listen()` +- :meth:`~socket.socket.makefile()` +- :meth:`~socket.socket.recv()`, :meth:`~socket.socket.recv_into()` + (but passing a non-zero ``flags`` argument is not allowed) +- :meth:`~socket.socket.send()`, :meth:`~socket.socket.sendall()` (with + the same limitation) +- :meth:`~socket.socket.shutdown()` + +However, since the SSL (and TLS) protocol has its own framing atop +of TCP, the SSL sockets abstraction can, in certain respects, diverge from +the specification of normal, OS-level sockets. + +SSL sockets also have the following additional methods and attributes: .. method:: SSLSocket.getpeercert(binary_form=False) From fdf04757a31639128737832952b06c26a754f2fb Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 6 Apr 2013 09:56:41 -0400 Subject: [PATCH 011/480] update pydoc-topics --- Lib/pydoc_data/topics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 63926d38bc1..75a71ff85cb 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Sat Mar 23 10:09:08 2013 +# Autogenerated by Sphinx on Sat Apr 6 09:55:30 2013 topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, ``assert expression``, is equivalent to\n\n if __debug__:\n if not expression: raise AssertionError\n\nThe extended form, ``assert expression1, expression2``, is equivalent\nto\n\n if __debug__:\n if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that ``__debug__`` and ``AssertionError``\nrefer to the built-in variables with those names. In the current\nimplementation, the built-in variable ``__debug__`` is ``True`` under\nnormal circumstances, ``False`` when optimization is requested\n(command line option -O). The current code generator emits no code\nfor an assert statement when optimization is requested at compile\ntime. Note that it is unnecessary to include the source code for the\nexpression that failed in the error message; it will be displayed as\npart of the stack trace.\n\nAssignments to ``__debug__`` are illegal. The value for the built-in\nvariable is determined when the interpreter starts.\n', 'assignment': '\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list is recursively defined as\nfollows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` statement in the\n current code block: the name is bound to the object in the current\n local namespace.\n\n * Otherwise: the name is bound to the object in the current global\n namespace.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield a plain integer. If it is negative, the\n sequence\'s length is added to it. The resulting value must be a\n nonnegative integer less than the sequence\'s length, and the\n sequence is asked to assign the assigned object to its item with\n that index. If the index is out of range, ``IndexError`` is raised\n (assignment to a subscripted sequence cannot add new items to a\n list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to (small) integers. If either\n bound is negative, the sequence\'s length is added to it. The\n resulting bounds are clipped to lie between zero and the sequence\'s\n length, inclusive. Finally, the sequence object is asked to replace\n the slice with the items of the assigned sequence. The length of\n the slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print x\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', 'atom-identifiers': '\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name in front of the name, with leading underscores removed, and\na single underscore inserted in front of the class name. For example,\nthe identifier ``__spam`` occurring in a class named ``Ham`` will be\ntransformed to ``_Ham__spam``. This transformation is independent of\nthe syntactical context in which the identifier is used. If the\ntransformed name is extremely long (longer than 255 characters),\nimplementation defined truncation may happen. If the class name\nconsists only of underscores, no transformation is done.\n', From c86a0249ac8121dfbabf72b9a3327a83b3143984 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 6 Apr 2013 09:58:51 -0400 Subject: [PATCH 012/480] change version to 2.7.4 --- Include/patchlevel.h | 6 +++--- Lib/distutils/__init__.py | 2 +- Lib/idlelib/idlever.py | 2 +- Misc/NEWS | 2 +- Misc/RPM/python-2.7.spec | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 7d6952e1d09..6b5e56645c2 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,11 +23,11 @@ #define PY_MAJOR_VERSION 2 #define PY_MINOR_VERSION 7 #define PY_MICRO_VERSION 4 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "2.7.4rc1+" +#define PY_VERSION "2.7.4" /*--end constants--*/ /* Subversion Revision number of this file (not of the repository). Empty diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py index 15b511bee58..84dbc3806a7 100644 --- a/Lib/distutils/__init__.py +++ b/Lib/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.4rc1" +__version__ = "2.7.4" #--end constants-- diff --git a/Lib/idlelib/idlever.py b/Lib/idlelib/idlever.py index 2aea6e02613..77421d5f770 100644 --- a/Lib/idlelib/idlever.py +++ b/Lib/idlelib/idlever.py @@ -1 +1 @@ -IDLE_VERSION = "2.7.4rc1" +IDLE_VERSION = "2.7.4" diff --git a/Misc/NEWS b/Misc/NEWS index 6f4740c80bb..c33be8b2b09 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -4,7 +4,7 @@ Python News What's New in Python 2.7.4? =========================== -*Release date: XXXX-XX-XX* +*Release date: 2013-04-06* Build ----- diff --git a/Misc/RPM/python-2.7.spec b/Misc/RPM/python-2.7.spec index 0b62adfe738..c00ba96cbd6 100644 --- a/Misc/RPM/python-2.7.spec +++ b/Misc/RPM/python-2.7.spec @@ -39,7 +39,7 @@ %define name python #--start constants-- -%define version 2.7.4rc1 +%define version 2.7.4 %define libvers 2.7 #--end constants-- %define release 1pydotorg From e3acdda5c2c98a48dd670eb4c5dceac53941c9fc Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 6 Apr 2013 09:59:25 -0400 Subject: [PATCH 013/480] Added tag v2.7.4 for changeset 026ee0057e2d From 1004e461311e03ae6f811d2723473fa029d411b2 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 6 Apr 2013 10:17:54 -0400 Subject: [PATCH 014/480] start cracking on 2.7.5 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 6b5e56645c2..b62d890bdc8 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -27,7 +27,7 @@ #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "2.7.4" +#define PY_VERSION "2.7.4+" /*--end constants--*/ /* Subversion Revision number of this file (not of the repository). Empty From 16e70e072e10e949dff0768b3d84cd5f272c6d51 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 6 Apr 2013 18:54:19 +0300 Subject: [PATCH 015/480] Change wording as Eli Bendersky suggests. --- Doc/howto/argparse.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 3d49d359292..4f64948c9d2 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -668,8 +668,8 @@ Conflicting options So far, we have been working with two methods of an :class:`argparse.ArgumentParser` instance. Let's introduce a third one, :meth:`add_mutually_exclusive_group`. It allows for us to specify options that -conflict with each other. Let's also change the rest of the program to make the -new functionality makes more sense: +conflict with each other. Let's also change the rest of the program so that +the new functionality makes more sense: we'll introduce the ``--quiet`` option, which will be the opposite of the ``--verbose`` one:: From 916883a2ef4105291722b7d8014a5bc0b2f4a2be Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 6 Apr 2013 22:04:10 +0300 Subject: [PATCH 016/480] Issue #14010: Fix a crash when iterating or deleting deeply nested filters in itertools module (i.e. itertools.izip(), itertools.chain(), etc). --- Lib/test/test_itertools.py | 135 ++++++++++++++++++++++++++++++++++++- Misc/NEWS | 4 +- Modules/itertoolsmodule.c | 75 ++++++++++++++++++--- Objects/abstract.c | 3 + 4 files changed, 205 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index db9f437d37a..dc1764131e0 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1413,6 +1413,139 @@ def __init__(self, newarg=None, *args): self.assertNotIn("does not take keyword arguments", err.args[0]) +class TestRecursionLimit(unittest.TestCase): + # Issue #14010 + recursionlimit = sys.getrecursionlimit() + + def test_chain(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = chain(it, ()) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_compress(self): + data = (0, 1) + selectors = (True, True) + it = data + for _ in xrange(self.recursionlimit): + it = compress(it, selectors) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + it = selectors + for _ in xrange(self.recursionlimit): + it = compress(data, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_cycle(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = cycle(it) + with self.assertRaises(RuntimeError): + for _ in range(3): + next(it) + del it + + def test_dropwhile(self): + it = (0, 1, 0) + for _ in xrange(self.recursionlimit): + it = dropwhile(bool, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_ifilter(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = ifilter(bool, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_ifilterfalse(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = ifilterfalse(bool, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_groupby(self): + key = operator.itemgetter(0) + it = ((0, []), (1, [])) + for _ in xrange(self.recursionlimit): + it = groupby(it, key) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_imap(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = imap(int, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_islice(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = islice(it, 2) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_starmap(self): + it = 'ab' + for _ in xrange(self.recursionlimit): + it = starmap(tuple, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_takewhile(self): + it = (1, 0) + for _ in xrange(self.recursionlimit): + it = takewhile(bool, it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_izip(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = izip(it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + def test_izip_longest(self): + it = (0, 1) + for _ in xrange(self.recursionlimit): + it = izip_longest(it) + with self.assertRaises(RuntimeError): + for _ in it: + pass + del it + + libreftest = """ Doctest for examples in the library reference: libitertools.tex @@ -1645,7 +1778,7 @@ def __init__(self, newarg=None, *args): def test_main(verbose=None): test_classes = (TestBasicOps, TestVariousIteratorArgs, TestGC, RegressionTests, LengthTransparency, - SubclassWithKwargsTest, TestExamples) + SubclassWithKwargsTest, TestExamples, TestRecursionLimit) test_support.run_unittest(*test_classes) # verify reference counting diff --git a/Misc/NEWS b/Misc/NEWS index a41ff4a3539..6bb4215d029 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -11,6 +11,8 @@ Core and Builtins Library ------- +- Issue #14010: Fix a crash when iterating or deleting deeply nested filters + in itertools module (i.e. itertools.izip(), itertools.chain(), etc). - Issue #14254: IDLE now handles readline correctly across shell restarts. @@ -285,7 +287,7 @@ Library - Issue #12718: Fix interaction with winpdb overriding __import__ by setting importer attribute on BaseConfigurator instance. - + - Issue #17521: Corrected non-enabling of logger following two calls to fileConfig(). diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ea822913662..73c5e30db01 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -54,12 +54,14 @@ static void groupby_dealloc(groupbyobject *gbo) { PyObject_GC_UnTrack(gbo); + Py_TRASHCAN_SAFE_BEGIN(gbo) Py_XDECREF(gbo->it); Py_XDECREF(gbo->keyfunc); Py_XDECREF(gbo->tgtkey); Py_XDECREF(gbo->currkey); Py_XDECREF(gbo->currvalue); Py_TYPE(gbo)->tp_free(gbo); + Py_TRASHCAN_SAFE_END(gbo) } static int @@ -741,9 +743,11 @@ static void cycle_dealloc(cycleobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->saved); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -886,9 +890,11 @@ static void dropwhile_dealloc(dropwhileobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -909,7 +915,10 @@ dropwhile_next(dropwhileobject *lz) iternext = *Py_TYPE(it)->tp_iternext; for (;;) { + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; item = iternext(it); + Py_LeaveRecursiveCall(); if (item == NULL) return NULL; if (lz->start == 1) @@ -1030,9 +1039,11 @@ static void takewhile_dealloc(takewhileobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -1053,7 +1064,10 @@ takewhile_next(takewhileobject *lz) if (lz->stop == 1) return NULL; + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; item = (*Py_TYPE(it)->tp_iternext)(it); + Py_LeaveRecursiveCall(); if (item == NULL) return NULL; @@ -1221,8 +1235,10 @@ static void islice_dealloc(isliceobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -1243,7 +1259,10 @@ islice_next(isliceobject *lz) iternext = *Py_TYPE(it)->tp_iternext; while (lz->cnt < lz->next) { + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; item = iternext(it); + Py_LeaveRecursiveCall(); if (item == NULL) return NULL; Py_DECREF(item); @@ -1251,7 +1270,10 @@ islice_next(isliceobject *lz) } if (stop != -1 && lz->cnt >= stop) return NULL; + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; item = iternext(it); + Py_LeaveRecursiveCall(); if (item == NULL) return NULL; lz->cnt++; @@ -1364,9 +1386,11 @@ static void starmap_dealloc(starmapobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -1384,7 +1408,10 @@ starmap_next(starmapobject *lz) PyObject *result; PyObject *it = lz->it; + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; args = (*Py_TYPE(it)->tp_iternext)(it); + Py_LeaveRecursiveCall(); if (args == NULL) return NULL; if (!PyTuple_CheckExact(args)) { @@ -1509,9 +1536,11 @@ static void imap_dealloc(imapobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->iters); Py_XDECREF(lz->func); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -1686,9 +1715,11 @@ static void chain_dealloc(chainobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->active); Py_XDECREF(lz->source); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -2837,9 +2868,11 @@ static void compress_dealloc(compressobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->data); Py_XDECREF(lz->selectors); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -2866,11 +2899,16 @@ compress_next(compressobject *lz) exception first). */ + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; datum = datanext(data); - if (datum == NULL) + if (datum == NULL) { + Py_LeaveRecursiveCall(); return NULL; + } selector = selectornext(selectors); + Py_LeaveRecursiveCall(); if (selector == NULL) { Py_DECREF(datum); return NULL; @@ -2983,9 +3021,11 @@ static void ifilter_dealloc(ifilterobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -3006,7 +3046,10 @@ ifilter_next(ifilterobject *lz) iternext = *Py_TYPE(it)->tp_iternext; for (;;) { + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; item = iternext(it); + Py_LeaveRecursiveCall(); if (item == NULL) return NULL; @@ -3128,9 +3171,11 @@ static void ifilterfalse_dealloc(ifilterfalseobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -3151,7 +3196,10 @@ ifilterfalse_next(ifilterfalseobject *lz) iternext = *Py_TYPE(it)->tp_iternext; for (;;) { + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; item = iternext(it); + Py_LeaveRecursiveCall(); if (item == NULL) return NULL; @@ -3551,9 +3599,11 @@ static void izip_dealloc(izipobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->ittuple); Py_XDECREF(lz->result); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int @@ -3576,15 +3626,15 @@ izip_next(izipobject *lz) if (tuplesize == 0) return NULL; + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; if (Py_REFCNT(result) == 1) { Py_INCREF(result); for (i=0 ; i < tuplesize ; i++) { it = PyTuple_GET_ITEM(lz->ittuple, i); item = (*Py_TYPE(it)->tp_iternext)(it); - if (item == NULL) { - Py_DECREF(result); - return NULL; - } + if (item == NULL) + goto error; olditem = PyTuple_GET_ITEM(result, i); PyTuple_SET_ITEM(result, i, item); Py_DECREF(olditem); @@ -3592,18 +3642,21 @@ izip_next(izipobject *lz) } else { result = PyTuple_New(tuplesize); if (result == NULL) - return NULL; + goto error; for (i=0 ; i < tuplesize ; i++) { it = PyTuple_GET_ITEM(lz->ittuple, i); item = (*Py_TYPE(it)->tp_iternext)(it); - if (item == NULL) { - Py_DECREF(result); - return NULL; - } + if (item == NULL) + goto error; PyTuple_SET_ITEM(result, i, item); } } + Py_LeaveRecursiveCall(); return result; +error: + Py_XDECREF(result); + Py_LeaveRecursiveCall(); + return NULL; } PyDoc_STRVAR(izip_doc, @@ -3892,10 +3945,12 @@ static void izip_longest_dealloc(iziplongestobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->ittuple); Py_XDECREF(lz->result); Py_XDECREF(lz->fillvalue); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_SAFE_END(lz) } static int diff --git a/Objects/abstract.c b/Objects/abstract.c index 3c887116636..bc247e594fb 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -3104,7 +3104,10 @@ PyObject * PyIter_Next(PyObject *iter) { PyObject *result; + if (Py_EnterRecursiveCall(" while iterating")) + return NULL; result = (*iter->ob_type->tp_iternext)(iter); + Py_LeaveRecursiveCall(); if (result == NULL && PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) From fcb965bb24dbdf953d8a0d12699d48a02dd3b0b3 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 6 Apr 2013 15:19:04 -0400 Subject: [PATCH 017/480] move idle news to its own section --- Misc/NEWS | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index a41ff4a3539..7f165dbdeec 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,15 +12,9 @@ Core and Builtins Library ------- -- Issue #14254: IDLE now handles readline correctly across shell restarts. - -- Issue #17614: IDLE no longer raises exception when quickly closing a file. - - Issue #13163: Rename operands in smtplib.SMTP._get_socket to correct names; fixes otherwise misleading output in tracebacks and when when debug is on. -- Issue #6698: IDLE now opens just an editor window when configured to do so. - - Issue #8900: Using keyboard shortcuts in IDLE to open a file no longer raises an exception. @@ -29,6 +23,15 @@ Library - Issue #17526: fix an IndexError raised while passing code without filename to inspect.findsource(). Initial patch by Tyler Doyle. +IDLE +---- + +- Issue #14254: IDLE now handles readline correctly across shell restarts. + +- Issue #17614: IDLE no longer raises exception when quickly closing a file. + +- Issue #6698: IDLE now opens just an editor window when configured to do so. + Documentation ------------- From 3a3e6f55548445a4ebebed81534241adbd287312 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 6 Apr 2013 21:21:04 +0200 Subject: [PATCH 018/480] Issue #17645: convert an assert() into a proper exception in _Py_Mangle(). --- Python/compile.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 531bed43c48..8354c75b393 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -221,8 +221,11 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident) } plen = strlen(p); - assert(1 <= PY_SSIZE_T_MAX - nlen); - assert(1 + nlen <= PY_SSIZE_T_MAX - plen); + if (plen + nlen >= PY_SSIZE_T_MAX - 1) { + PyErr_SetString(PyExc_OverflowError, + "private identifier too large to be mangled"); + return NULL; + } ident = PyString_FromStringAndSize(NULL, 1 + nlen + plen); if (!ident) From e7c3c7a0d200f0e6462909f41500b26ac7642132 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 6 Apr 2013 22:51:29 +0300 Subject: [PATCH 019/480] Revert a premature patch for issue #14010 (changeset d17d10c84d27). --- Lib/test/test_itertools.py | 135 +------------------------------------ Misc/NEWS | 4 +- Modules/itertoolsmodule.c | 75 +++------------------ Objects/abstract.c | 3 - 4 files changed, 12 insertions(+), 205 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index dc1764131e0..db9f437d37a 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1413,139 +1413,6 @@ def __init__(self, newarg=None, *args): self.assertNotIn("does not take keyword arguments", err.args[0]) -class TestRecursionLimit(unittest.TestCase): - # Issue #14010 - recursionlimit = sys.getrecursionlimit() - - def test_chain(self): - it = (0, 1) - for _ in xrange(self.recursionlimit): - it = chain(it, ()) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - def test_compress(self): - data = (0, 1) - selectors = (True, True) - it = data - for _ in xrange(self.recursionlimit): - it = compress(it, selectors) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - it = selectors - for _ in xrange(self.recursionlimit): - it = compress(data, it) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - def test_cycle(self): - it = (0, 1) - for _ in xrange(self.recursionlimit): - it = cycle(it) - with self.assertRaises(RuntimeError): - for _ in range(3): - next(it) - del it - - def test_dropwhile(self): - it = (0, 1, 0) - for _ in xrange(self.recursionlimit): - it = dropwhile(bool, it) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - def test_ifilter(self): - it = (0, 1) - for _ in xrange(self.recursionlimit): - it = ifilter(bool, it) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - def test_ifilterfalse(self): - it = (0, 1) - for _ in xrange(self.recursionlimit): - it = ifilterfalse(bool, it) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - def test_groupby(self): - key = operator.itemgetter(0) - it = ((0, []), (1, [])) - for _ in xrange(self.recursionlimit): - it = groupby(it, key) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - def test_imap(self): - it = (0, 1) - for _ in xrange(self.recursionlimit): - it = imap(int, it) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - def test_islice(self): - it = (0, 1) - for _ in xrange(self.recursionlimit): - it = islice(it, 2) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - def test_starmap(self): - it = 'ab' - for _ in xrange(self.recursionlimit): - it = starmap(tuple, it) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - def test_takewhile(self): - it = (1, 0) - for _ in xrange(self.recursionlimit): - it = takewhile(bool, it) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - def test_izip(self): - it = (0, 1) - for _ in xrange(self.recursionlimit): - it = izip(it) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - def test_izip_longest(self): - it = (0, 1) - for _ in xrange(self.recursionlimit): - it = izip_longest(it) - with self.assertRaises(RuntimeError): - for _ in it: - pass - del it - - libreftest = """ Doctest for examples in the library reference: libitertools.tex @@ -1778,7 +1645,7 @@ def test_izip_longest(self): def test_main(verbose=None): test_classes = (TestBasicOps, TestVariousIteratorArgs, TestGC, RegressionTests, LengthTransparency, - SubclassWithKwargsTest, TestExamples, TestRecursionLimit) + SubclassWithKwargsTest, TestExamples) test_support.run_unittest(*test_classes) # verify reference counting diff --git a/Misc/NEWS b/Misc/NEWS index 79f49569b0d..7f165dbdeec 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -11,8 +11,6 @@ Core and Builtins Library ------- -- Issue #14010: Fix a crash when iterating or deleting deeply nested filters - in itertools module (i.e. itertools.izip(), itertools.chain(), etc). - Issue #13163: Rename operands in smtplib.SMTP._get_socket to correct names; fixes otherwise misleading output in tracebacks and when when debug is on. @@ -290,7 +288,7 @@ Library - Issue #12718: Fix interaction with winpdb overriding __import__ by setting importer attribute on BaseConfigurator instance. - + - Issue #17521: Corrected non-enabling of logger following two calls to fileConfig(). diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 73c5e30db01..ea822913662 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -54,14 +54,12 @@ static void groupby_dealloc(groupbyobject *gbo) { PyObject_GC_UnTrack(gbo); - Py_TRASHCAN_SAFE_BEGIN(gbo) Py_XDECREF(gbo->it); Py_XDECREF(gbo->keyfunc); Py_XDECREF(gbo->tgtkey); Py_XDECREF(gbo->currkey); Py_XDECREF(gbo->currvalue); Py_TYPE(gbo)->tp_free(gbo); - Py_TRASHCAN_SAFE_END(gbo) } static int @@ -743,11 +741,9 @@ static void cycle_dealloc(cycleobject *lz) { PyObject_GC_UnTrack(lz); - Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->saved); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); - Py_TRASHCAN_SAFE_END(lz) } static int @@ -890,11 +886,9 @@ static void dropwhile_dealloc(dropwhileobject *lz) { PyObject_GC_UnTrack(lz); - Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); - Py_TRASHCAN_SAFE_END(lz) } static int @@ -915,10 +909,7 @@ dropwhile_next(dropwhileobject *lz) iternext = *Py_TYPE(it)->tp_iternext; for (;;) { - if (Py_EnterRecursiveCall(" while iterating")) - return NULL; item = iternext(it); - Py_LeaveRecursiveCall(); if (item == NULL) return NULL; if (lz->start == 1) @@ -1039,11 +1030,9 @@ static void takewhile_dealloc(takewhileobject *lz) { PyObject_GC_UnTrack(lz); - Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); - Py_TRASHCAN_SAFE_END(lz) } static int @@ -1064,10 +1053,7 @@ takewhile_next(takewhileobject *lz) if (lz->stop == 1) return NULL; - if (Py_EnterRecursiveCall(" while iterating")) - return NULL; item = (*Py_TYPE(it)->tp_iternext)(it); - Py_LeaveRecursiveCall(); if (item == NULL) return NULL; @@ -1235,10 +1221,8 @@ static void islice_dealloc(isliceobject *lz) { PyObject_GC_UnTrack(lz); - Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); - Py_TRASHCAN_SAFE_END(lz) } static int @@ -1259,10 +1243,7 @@ islice_next(isliceobject *lz) iternext = *Py_TYPE(it)->tp_iternext; while (lz->cnt < lz->next) { - if (Py_EnterRecursiveCall(" while iterating")) - return NULL; item = iternext(it); - Py_LeaveRecursiveCall(); if (item == NULL) return NULL; Py_DECREF(item); @@ -1270,10 +1251,7 @@ islice_next(isliceobject *lz) } if (stop != -1 && lz->cnt >= stop) return NULL; - if (Py_EnterRecursiveCall(" while iterating")) - return NULL; item = iternext(it); - Py_LeaveRecursiveCall(); if (item == NULL) return NULL; lz->cnt++; @@ -1386,11 +1364,9 @@ static void starmap_dealloc(starmapobject *lz) { PyObject_GC_UnTrack(lz); - Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); - Py_TRASHCAN_SAFE_END(lz) } static int @@ -1408,10 +1384,7 @@ starmap_next(starmapobject *lz) PyObject *result; PyObject *it = lz->it; - if (Py_EnterRecursiveCall(" while iterating")) - return NULL; args = (*Py_TYPE(it)->tp_iternext)(it); - Py_LeaveRecursiveCall(); if (args == NULL) return NULL; if (!PyTuple_CheckExact(args)) { @@ -1536,11 +1509,9 @@ static void imap_dealloc(imapobject *lz) { PyObject_GC_UnTrack(lz); - Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->iters); Py_XDECREF(lz->func); Py_TYPE(lz)->tp_free(lz); - Py_TRASHCAN_SAFE_END(lz) } static int @@ -1715,11 +1686,9 @@ static void chain_dealloc(chainobject *lz) { PyObject_GC_UnTrack(lz); - Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->active); Py_XDECREF(lz->source); Py_TYPE(lz)->tp_free(lz); - Py_TRASHCAN_SAFE_END(lz) } static int @@ -2868,11 +2837,9 @@ static void compress_dealloc(compressobject *lz) { PyObject_GC_UnTrack(lz); - Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->data); Py_XDECREF(lz->selectors); Py_TYPE(lz)->tp_free(lz); - Py_TRASHCAN_SAFE_END(lz) } static int @@ -2899,16 +2866,11 @@ compress_next(compressobject *lz) exception first). */ - if (Py_EnterRecursiveCall(" while iterating")) - return NULL; datum = datanext(data); - if (datum == NULL) { - Py_LeaveRecursiveCall(); + if (datum == NULL) return NULL; - } selector = selectornext(selectors); - Py_LeaveRecursiveCall(); if (selector == NULL) { Py_DECREF(datum); return NULL; @@ -3021,11 +2983,9 @@ static void ifilter_dealloc(ifilterobject *lz) { PyObject_GC_UnTrack(lz); - Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); - Py_TRASHCAN_SAFE_END(lz) } static int @@ -3046,10 +3006,7 @@ ifilter_next(ifilterobject *lz) iternext = *Py_TYPE(it)->tp_iternext; for (;;) { - if (Py_EnterRecursiveCall(" while iterating")) - return NULL; item = iternext(it); - Py_LeaveRecursiveCall(); if (item == NULL) return NULL; @@ -3171,11 +3128,9 @@ static void ifilterfalse_dealloc(ifilterfalseobject *lz) { PyObject_GC_UnTrack(lz); - Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); - Py_TRASHCAN_SAFE_END(lz) } static int @@ -3196,10 +3151,7 @@ ifilterfalse_next(ifilterfalseobject *lz) iternext = *Py_TYPE(it)->tp_iternext; for (;;) { - if (Py_EnterRecursiveCall(" while iterating")) - return NULL; item = iternext(it); - Py_LeaveRecursiveCall(); if (item == NULL) return NULL; @@ -3599,11 +3551,9 @@ static void izip_dealloc(izipobject *lz) { PyObject_GC_UnTrack(lz); - Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->ittuple); Py_XDECREF(lz->result); Py_TYPE(lz)->tp_free(lz); - Py_TRASHCAN_SAFE_END(lz) } static int @@ -3626,15 +3576,15 @@ izip_next(izipobject *lz) if (tuplesize == 0) return NULL; - if (Py_EnterRecursiveCall(" while iterating")) - return NULL; if (Py_REFCNT(result) == 1) { Py_INCREF(result); for (i=0 ; i < tuplesize ; i++) { it = PyTuple_GET_ITEM(lz->ittuple, i); item = (*Py_TYPE(it)->tp_iternext)(it); - if (item == NULL) - goto error; + if (item == NULL) { + Py_DECREF(result); + return NULL; + } olditem = PyTuple_GET_ITEM(result, i); PyTuple_SET_ITEM(result, i, item); Py_DECREF(olditem); @@ -3642,21 +3592,18 @@ izip_next(izipobject *lz) } else { result = PyTuple_New(tuplesize); if (result == NULL) - goto error; + return NULL; for (i=0 ; i < tuplesize ; i++) { it = PyTuple_GET_ITEM(lz->ittuple, i); item = (*Py_TYPE(it)->tp_iternext)(it); - if (item == NULL) - goto error; + if (item == NULL) { + Py_DECREF(result); + return NULL; + } PyTuple_SET_ITEM(result, i, item); } } - Py_LeaveRecursiveCall(); return result; -error: - Py_XDECREF(result); - Py_LeaveRecursiveCall(); - return NULL; } PyDoc_STRVAR(izip_doc, @@ -3945,12 +3892,10 @@ static void izip_longest_dealloc(iziplongestobject *lz) { PyObject_GC_UnTrack(lz); - Py_TRASHCAN_SAFE_BEGIN(lz) Py_XDECREF(lz->ittuple); Py_XDECREF(lz->result); Py_XDECREF(lz->fillvalue); Py_TYPE(lz)->tp_free(lz); - Py_TRASHCAN_SAFE_END(lz) } static int diff --git a/Objects/abstract.c b/Objects/abstract.c index bc247e594fb..3c887116636 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -3104,10 +3104,7 @@ PyObject * PyIter_Next(PyObject *iter) { PyObject *result; - if (Py_EnterRecursiveCall(" while iterating")) - return NULL; result = (*iter->ob_type->tp_iternext)(iter); - Py_LeaveRecursiveCall(); if (result == NULL && PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) From 39c232dde89cba55f074fece42deb4c27812f5fc Mon Sep 17 00:00:00 2001 From: Roger Serwy Date: Sat, 6 Apr 2013 20:21:21 -0500 Subject: [PATCH 020/480] move idle news entries to its section --- Misc/NEWS | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 7f165dbdeec..5ec6f461eee 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -15,11 +15,6 @@ Library - Issue #13163: Rename operands in smtplib.SMTP._get_socket to correct names; fixes otherwise misleading output in tracebacks and when when debug is on. -- Issue #8900: Using keyboard shortcuts in IDLE to open a file no longer - raises an exception. - -- Issue #6649: Fixed missing exit status in IDLE. Patch by Guilherme Polo. - - Issue #17526: fix an IndexError raised while passing code without filename to inspect.findsource(). Initial patch by Tyler Doyle. @@ -32,6 +27,12 @@ IDLE - Issue #6698: IDLE now opens just an editor window when configured to do so. +- Issue #8900: Using keyboard shortcuts in IDLE to open a file no longer + raises an exception. + +- Issue #6649: Fixed missing exit status in IDLE. Patch by Guilherme Polo. + + Documentation ------------- From 791f0f01e16b233e1432d9d1303bb72840f62ae4 Mon Sep 17 00:00:00 2001 From: Roger Serwy Date: Sat, 6 Apr 2013 20:26:53 -0500 Subject: [PATCH 021/480] #16887: IDLE now accepts Cancel in tabify/untabify dialog box. --- Lib/idlelib/EditorWindow.py | 4 +++- Misc/NEWS | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/idlelib/EditorWindow.py b/Lib/idlelib/EditorWindow.py index 0a01c9ec305..96f1ee58784 100644 --- a/Lib/idlelib/EditorWindow.py +++ b/Lib/idlelib/EditorWindow.py @@ -1425,6 +1425,7 @@ def uncomment_region_event(self, event): def tabify_region_event(self, event): head, tail, chars, lines = self.get_region() tabwidth = self._asktabwidth() + if tabwidth is None: return for pos in range(len(lines)): line = lines[pos] if line: @@ -1436,6 +1437,7 @@ def tabify_region_event(self, event): def untabify_region_event(self, event): head, tail, chars, lines = self.get_region() tabwidth = self._asktabwidth() + if tabwidth is None: return for pos in range(len(lines)): lines[pos] = lines[pos].expandtabs(tabwidth) self.set_region(head, tail, chars, lines) @@ -1529,7 +1531,7 @@ def _asktabwidth(self): parent=self.text, initialvalue=self.indentwidth, minvalue=2, - maxvalue=16) or self.tabwidth + maxvalue=16) # Guess indentwidth from text content. # Return guessed indentwidth. This should not be believed unless diff --git a/Misc/NEWS b/Misc/NEWS index 5ec6f461eee..9f145c770a2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,8 @@ Library IDLE ---- +- Issue #16887: IDLE now accepts Cancel in tabify/untabify dialog box. + - Issue #14254: IDLE now handles readline correctly across shell restarts. - Issue #17614: IDLE no longer raises exception when quickly closing a file. From 76baf786dcd4c81b5dd3b07152cf4966fe011e2f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 7 Apr 2013 14:45:37 +0300 Subject: [PATCH 022/480] Update argparse docs to follow order of ArgumentParser() arguments. --- Doc/library/argparse.rst | 405 ++++++++++++++++++++------------------- Misc/ACKS | 1 + 2 files changed, 204 insertions(+), 202 deletions(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index d18b51b6d55..61cb499b132 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -137,40 +137,136 @@ ArgumentParser objects argument_default=None, conflict_handler='error', \ add_help=True) - Create a new :class:`ArgumentParser` object. Each parameter has its own more - detailed description below, but in short they are: + Create a new :class:`ArgumentParser` object. All parameters should be passed + as keyword arguments. Each parameter has its own more detailed description + below, but in short they are: - * description_ - Text to display before the argument help. + * prog_ - The name of the program (default: ``sys.argv[0]``) - * epilog_ - Text to display after the argument help. + * usage_ - The string describing the program usage (default: generated from + arguments added to parser) - * add_help_ - Add a -h/--help option to the parser. (default: ``True``) + * description_ - Text to display before the argument help (default: none) - * argument_default_ - Set the global default value for arguments. - (default: ``None``) + * epilog_ - Text to display after the argument help (default: none) * parents_ - A list of :class:`ArgumentParser` objects whose arguments should - also be included. + also be included + + * formatter_class_ - A class for customizing the help output - * prefix_chars_ - The set of characters that prefix optional arguments. + * prefix_chars_ - The set of characters that prefix optional arguments (default: '-') * fromfile_prefix_chars_ - The set of characters that prefix files from - which additional arguments should be read. (default: ``None``) - - * formatter_class_ - A class for customizing the help output. + which additional arguments should be read (default: ``None``) - * conflict_handler_ - Usually unnecessary, defines strategy for resolving - conflicting optionals. + * argument_default_ - The global default value for arguments + (default: ``None``) - * prog_ - The name of the program (default: - ``sys.argv[0]``) + * conflict_handler_ - The strategy for resolving conflicting optionals + (usually unnecessary) - * usage_ - The string describing the program usage (default: generated) + * add_help_ - Add a -h/--help option to the parser (default: ``True``) The following sections describe how each of these are used. +prog +^^^^ + +By default, :class:`ArgumentParser` objects uses ``sys.argv[0]`` to determine +how to display the name of the program in help messages. This default is almost +always desirable because it will make the help messages match how the program was +invoked on the command line. For example, consider a file named +``myprogram.py`` with the following code:: + + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--foo', help='foo help') + args = parser.parse_args() + +The help for this program will display ``myprogram.py`` as the program name +(regardless of where the program was invoked from):: + + $ python myprogram.py --help + usage: myprogram.py [-h] [--foo FOO] + + optional arguments: + -h, --help show this help message and exit + --foo FOO foo help + $ cd .. + $ python subdir\myprogram.py --help + usage: myprogram.py [-h] [--foo FOO] + + optional arguments: + -h, --help show this help message and exit + --foo FOO foo help + +To change this default behavior, another value can be supplied using the +``prog=`` argument to :class:`ArgumentParser`:: + + >>> parser = argparse.ArgumentParser(prog='myprogram') + >>> parser.print_help() + usage: myprogram [-h] + + optional arguments: + -h, --help show this help message and exit + +Note that the program name, whether determined from ``sys.argv[0]`` or from the +``prog=`` argument, is available to help messages using the ``%(prog)s`` format +specifier. + +:: + + >>> parser = argparse.ArgumentParser(prog='myprogram') + >>> parser.add_argument('--foo', help='foo of the %(prog)s program') + >>> parser.print_help() + usage: myprogram [-h] [--foo FOO] + + optional arguments: + -h, --help show this help message and exit + --foo FOO foo of the myprogram program + + +usage +^^^^^ + +By default, :class:`ArgumentParser` calculates the usage message from the +arguments it contains:: + + >>> parser = argparse.ArgumentParser(prog='PROG') + >>> parser.add_argument('--foo', nargs='?', help='foo help') + >>> parser.add_argument('bar', nargs='+', help='bar help') + >>> parser.print_help() + usage: PROG [-h] [--foo [FOO]] bar [bar ...] + + positional arguments: + bar bar help + + optional arguments: + -h, --help show this help message and exit + --foo [FOO] foo help + +The default message can be overridden with the ``usage=`` keyword argument:: + + >>> parser = argparse.ArgumentParser(prog='PROG', usage='%(prog)s [options]') + >>> parser.add_argument('--foo', nargs='?', help='foo help') + >>> parser.add_argument('bar', nargs='+', help='bar help') + >>> parser.print_help() + usage: PROG [options] + + positional arguments: + bar bar help + + optional arguments: + -h, --help show this help message and exit + --foo [FOO] foo help + +The ``%(prog)s`` format specifier is available to fill in the program name in +your usage messages. + + description ^^^^^^^^^^^ @@ -218,122 +314,6 @@ line-wrapped, but this behavior can be adjusted with the formatter_class_ argument to :class:`ArgumentParser`. -add_help -^^^^^^^^ - -By default, ArgumentParser objects add an option which simply displays -the parser's help message. For example, consider a file named -``myprogram.py`` containing the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() - -If ``-h`` or ``--help`` is supplied at the command line, the ArgumentParser -help will be printed:: - - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - --foo FOO foo help - -Occasionally, it may be useful to disable the addition of this help option. -This can be achieved by passing ``False`` as the ``add_help=`` argument to -:class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) - >>> parser.add_argument('--foo', help='foo help') - >>> parser.print_help() - usage: PROG [--foo FOO] - - optional arguments: - --foo FOO foo help - -The help option is typically ``-h/--help``. The exception to this is -if the ``prefix_chars=`` is specified and does not include ``-``, in -which case ``-h`` and ``--help`` are not valid options. In -this case, the first character in ``prefix_chars`` is used to prefix -the help options:: - - >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='+/') - >>> parser.print_help() - usage: PROG [+h] - - optional arguments: - +h, ++help show this help message and exit - - -prefix_chars -^^^^^^^^^^^^ - -Most command-line options will use ``-`` as the prefix, e.g. ``-f/--foo``. -Parsers that need to support different or additional prefix -characters, e.g. for options -like ``+f`` or ``/foo``, may specify them using the ``prefix_chars=`` argument -to the ArgumentParser constructor:: - - >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='-+') - >>> parser.add_argument('+f') - >>> parser.add_argument('++bar') - >>> parser.parse_args('+f X ++bar Y'.split()) - Namespace(bar='Y', f='X') - -The ``prefix_chars=`` argument defaults to ``'-'``. Supplying a set of -characters that does not include ``-`` will cause ``-f/--foo`` options to be -disallowed. - - -fromfile_prefix_chars -^^^^^^^^^^^^^^^^^^^^^ - -Sometimes, for example when dealing with a particularly long argument lists, it -may make sense to keep the list of arguments in a file rather than typing it out -at the command line. If the ``fromfile_prefix_chars=`` argument is given to the -:class:`ArgumentParser` constructor, then arguments that start with any of the -specified characters will be treated as files, and will be replaced by the -arguments they contain. For example:: - - >>> with open('args.txt', 'w') as fp: - ... fp.write('-f\nbar') - >>> parser = argparse.ArgumentParser(fromfile_prefix_chars='@') - >>> parser.add_argument('-f') - >>> parser.parse_args(['-f', 'foo', '@args.txt']) - Namespace(f='bar') - -Arguments read from a file must by default be one per line (but see also -:meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they -were in the same place as the original file referencing argument on the command -line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` -is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. - -The ``fromfile_prefix_chars=`` argument defaults to ``None``, meaning that -arguments will never be treated as file references. - - -argument_default -^^^^^^^^^^^^^^^^ - -Generally, argument defaults are specified either by passing a default to -:meth:`~ArgumentParser.add_argument` or by calling the -:meth:`~ArgumentParser.set_defaults` methods with a specific set of name-value -pairs. Sometimes however, it may be useful to specify a single parser-wide -default for arguments. This can be accomplished by passing the -``argument_default=`` keyword argument to :class:`ArgumentParser`. For example, -to globally suppress attribute creation on :meth:`~ArgumentParser.parse_args` -calls, we supply ``argument_default=SUPPRESS``:: - - >>> parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS) - >>> parser.add_argument('--foo') - >>> parser.add_argument('bar', nargs='?') - >>> parser.parse_args(['--foo', '1', 'BAR']) - Namespace(bar='BAR', foo='1') - >>> parser.parse_args([]) - Namespace() - - parents ^^^^^^^ @@ -452,6 +432,74 @@ will add information about the default value of each of the arguments:: --foo FOO FOO! (default: 42) +prefix_chars +^^^^^^^^^^^^ + +Most command-line options will use ``-`` as the prefix, e.g. ``-f/--foo``. +Parsers that need to support different or additional prefix +characters, e.g. for options +like ``+f`` or ``/foo``, may specify them using the ``prefix_chars=`` argument +to the ArgumentParser constructor:: + + >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='-+') + >>> parser.add_argument('+f') + >>> parser.add_argument('++bar') + >>> parser.parse_args('+f X ++bar Y'.split()) + Namespace(bar='Y', f='X') + +The ``prefix_chars=`` argument defaults to ``'-'``. Supplying a set of +characters that does not include ``-`` will cause ``-f/--foo`` options to be +disallowed. + + +fromfile_prefix_chars +^^^^^^^^^^^^^^^^^^^^^ + +Sometimes, for example when dealing with a particularly long argument lists, it +may make sense to keep the list of arguments in a file rather than typing it out +at the command line. If the ``fromfile_prefix_chars=`` argument is given to the +:class:`ArgumentParser` constructor, then arguments that start with any of the +specified characters will be treated as files, and will be replaced by the +arguments they contain. For example:: + + >>> with open('args.txt', 'w') as fp: + ... fp.write('-f\nbar') + >>> parser = argparse.ArgumentParser(fromfile_prefix_chars='@') + >>> parser.add_argument('-f') + >>> parser.parse_args(['-f', 'foo', '@args.txt']) + Namespace(f='bar') + +Arguments read from a file must by default be one per line (but see also +:meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they +were in the same place as the original file referencing argument on the command +line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` +is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. + +The ``fromfile_prefix_chars=`` argument defaults to ``None``, meaning that +arguments will never be treated as file references. + + +argument_default +^^^^^^^^^^^^^^^^ + +Generally, argument defaults are specified either by passing a default to +:meth:`~ArgumentParser.add_argument` or by calling the +:meth:`~ArgumentParser.set_defaults` methods with a specific set of name-value +pairs. Sometimes however, it may be useful to specify a single parser-wide +default for arguments. This can be accomplished by passing the +``argument_default=`` keyword argument to :class:`ArgumentParser`. For example, +to globally suppress attribute creation on :meth:`~ArgumentParser.parse_args` +calls, we supply ``argument_default=SUPPRESS``:: + + >>> parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS) + >>> parser.add_argument('--foo') + >>> parser.add_argument('bar', nargs='?') + >>> parser.parse_args(['--foo', '1', 'BAR']) + Namespace(bar='BAR', foo='1') + >>> parser.parse_args([]) + Namespace() + + conflict_handler ^^^^^^^^^^^^^^^^ @@ -489,22 +537,20 @@ action is retained as the ``-f`` action, because only the ``--foo`` option string was overridden. -prog -^^^^ +add_help +^^^^^^^^ -By default, :class:`ArgumentParser` objects uses ``sys.argv[0]`` to determine -how to display the name of the program in help messages. This default is almost -always desirable because it will make the help messages match how the program was -invoked on the command line. For example, consider a file named -``myprogram.py`` with the following code:: +By default, ArgumentParser objects add an option which simply displays +the parser's help message. For example, consider a file named +``myprogram.py`` containing the following code:: import argparse parser = argparse.ArgumentParser() parser.add_argument('--foo', help='foo help') args = parser.parse_args() -The help for this program will display ``myprogram.py`` as the program name -(regardless of where the program was invoked from):: +If ``-h`` or ``--help`` is supplied at the command line, the ArgumentParser +help will be printed:: $ python myprogram.py --help usage: myprogram.py [-h] [--foo FOO] @@ -512,76 +558,31 @@ The help for this program will display ``myprogram.py`` as the program name optional arguments: -h, --help show this help message and exit --foo FOO foo help - $ cd .. - $ python subdir\myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - optional arguments: - -h, --help show this help message and exit - --foo FOO foo help - -To change this default behavior, another value can be supplied using the -``prog=`` argument to :class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser(prog='myprogram') - >>> parser.print_help() - usage: myprogram [-h] - - optional arguments: - -h, --help show this help message and exit - -Note that the program name, whether determined from ``sys.argv[0]`` or from the -``prog=`` argument, is available to help messages using the ``%(prog)s`` format -specifier. - -:: - - >>> parser = argparse.ArgumentParser(prog='myprogram') - >>> parser.add_argument('--foo', help='foo of the %(prog)s program') - >>> parser.print_help() - usage: myprogram [-h] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - --foo FOO foo of the myprogram program - - -usage -^^^^^ - -By default, :class:`ArgumentParser` calculates the usage message from the -arguments it contains:: +Occasionally, it may be useful to disable the addition of this help option. +This can be achieved by passing ``False`` as the ``add_help=`` argument to +:class:`ArgumentParser`:: - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') + >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) + >>> parser.add_argument('--foo', help='foo help') >>> parser.print_help() - usage: PROG [-h] [--foo [FOO]] bar [bar ...] - - positional arguments: - bar bar help + usage: PROG [--foo FOO] optional arguments: - -h, --help show this help message and exit - --foo [FOO] foo help + --foo FOO foo help -The default message can be overridden with the ``usage=`` keyword argument:: +The help option is typically ``-h/--help``. The exception to this is +if the ``prefix_chars=`` is specified and does not include ``-``, in +which case ``-h`` and ``--help`` are not valid options. In +this case, the first character in ``prefix_chars`` is used to prefix +the help options:: - >>> parser = argparse.ArgumentParser(prog='PROG', usage='%(prog)s [options]') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') + >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='+/') >>> parser.print_help() - usage: PROG [options] - - positional arguments: - bar bar help + usage: PROG [-h] optional arguments: - -h, --help show this help message and exit - --foo [FOO] foo help - -The ``%(prog)s`` format specifier is available to fill in the program name in -your usage messages. + -h, --help show this help message and exit The add_argument() method diff --git a/Misc/ACKS b/Misc/ACKS index a91dc327e94..82c8bc620b2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -593,6 +593,7 @@ Luke Kenneth Casson Leighton Tshepang Lekhonkhobe Marc-André Lemburg John Lenton +Kostyantyn Leschenko Christopher Tur Lesniewski-Laas Mark Levinson William Lewis From 567f59b9626685c442f1da603fa12631728f058a Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 7 Apr 2013 09:52:59 -0400 Subject: [PATCH 023/480] list slotdefs in offset order rather than sorting them (closes #17610) This means we can remove our usage of qsort() than relied on undefined behavior. Backport by Zbigniew Halas. --- Misc/ACKS | 1 + Misc/NEWS | 2 + Objects/typeobject.c | 245 ++++++++++++++++++++----------------------- 3 files changed, 117 insertions(+), 131 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS index 82c8bc620b2..12507e41203 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -377,6 +377,7 @@ Paul ten Hagen Rasmus Hahn Peter Haight Václav Haisman +Zbigniew Halas Bob Halley Jesse Hallio Jun Hamano diff --git a/Misc/NEWS b/Misc/NEWS index 9f145c770a2..3d9e9dcb581 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -9,6 +9,8 @@ What's New in Python 2.7.5? Core and Builtins ----------------- +- Issue #17610: Don't rely on non-standard behavior of the C qsort() function. + Library ------- diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 518d6e8e052..6c88f53af42 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5795,15 +5795,16 @@ slot_tp_del(PyObject *self) } -/* Table mapping __foo__ names to tp_foo offsets and slot_tp_foo wrapper - functions. The offsets here are relative to the 'PyHeapTypeObject' - structure, which incorporates the additional structures used for numbers, - sequences and mappings. - Note that multiple names may map to the same slot (e.g. __eq__, - __ne__ etc. all map to tp_richcompare) and one name may map to multiple - slots (e.g. __str__ affects tp_str as well as tp_repr). The table is - terminated with an all-zero entry. (This table is further initialized and - sorted in init_slotdefs() below.) */ +/* +Table mapping __foo__ names to tp_foo offsets and slot_tp_foo wrapper functions. + +The table is ordered by offsets relative to the 'PyHeapTypeObject' structure, +which incorporates the additional structures used for numbers, sequences and +mappings. Note that multiple names may map to the same slot (e.g. __eq__, +__ne__ etc. all map to tp_richcompare) and one name may map to multiple slots +(e.g. __str__ affects tp_str as well as tp_repr). The table is terminated with +an all-zero entry. (This table is further initialized in init_slotdefs().) +*/ typedef struct wrapperbase slotdef; @@ -5853,57 +5854,57 @@ typedef struct wrapperbase slotdef; "x." NAME "(y) <==> " DOC) static slotdef slotdefs[] = { - SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc, - "x.__len__() <==> len(x)"), - /* Heap types defining __add__/__mul__ have sq_concat/sq_repeat == NULL. - The logic in abstract.c always falls back to nb_add/nb_multiply in - this case. Defining both the nb_* and the sq_* slots to call the - user-defined methods has unexpected side-effects, as shown by - test_descr.notimplemented() */ - SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc, - "x.__add__(y) <==> x+y"), - SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc, - "x.__mul__(n) <==> x*n"), - SQSLOT("__rmul__", sq_repeat, NULL, wrap_indexargfunc, - "x.__rmul__(n) <==> n*x"), - SQSLOT("__getitem__", sq_item, slot_sq_item, wrap_sq_item, - "x.__getitem__(y) <==> x[y]"), - SQSLOT("__getslice__", sq_slice, slot_sq_slice, wrap_ssizessizeargfunc, - "x.__getslice__(i, j) <==> x[i:j]\n\ - \n\ - Use of negative indices is not supported."), - SQSLOT("__setitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_setitem, - "x.__setitem__(i, y) <==> x[i]=y"), - SQSLOT("__delitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_delitem, - "x.__delitem__(y) <==> del x[y]"), - SQSLOT("__setslice__", sq_ass_slice, slot_sq_ass_slice, - wrap_ssizessizeobjargproc, - "x.__setslice__(i, j, y) <==> x[i:j]=y\n\ - \n\ - Use of negative indices is not supported."), - SQSLOT("__delslice__", sq_ass_slice, slot_sq_ass_slice, wrap_delslice, - "x.__delslice__(i, j) <==> del x[i:j]\n\ - \n\ - Use of negative indices is not supported."), - SQSLOT("__contains__", sq_contains, slot_sq_contains, wrap_objobjproc, - "x.__contains__(y) <==> y in x"), - SQSLOT("__iadd__", sq_inplace_concat, NULL, - wrap_binaryfunc, "x.__iadd__(y) <==> x+=y"), - SQSLOT("__imul__", sq_inplace_repeat, NULL, - wrap_indexargfunc, "x.__imul__(y) <==> x*=y"), - - MPSLOT("__len__", mp_length, slot_mp_length, wrap_lenfunc, - "x.__len__() <==> len(x)"), - MPSLOT("__getitem__", mp_subscript, slot_mp_subscript, - wrap_binaryfunc, - "x.__getitem__(y) <==> x[y]"), - MPSLOT("__setitem__", mp_ass_subscript, slot_mp_ass_subscript, - wrap_objobjargproc, - "x.__setitem__(i, y) <==> x[i]=y"), - MPSLOT("__delitem__", mp_ass_subscript, slot_mp_ass_subscript, - wrap_delitem, - "x.__delitem__(y) <==> del x[y]"), - + TPSLOT("__str__", tp_print, NULL, NULL, ""), + TPSLOT("__repr__", tp_print, NULL, NULL, ""), + TPSLOT("__getattribute__", tp_getattr, NULL, NULL, ""), + TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""), + TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""), + TPSLOT("__delattr__", tp_setattr, NULL, NULL, ""), + TPSLOT("__cmp__", tp_compare, _PyObject_SlotCompare, wrap_cmpfunc, + "x.__cmp__(y) <==> cmp(x,y)"), + TPSLOT("__repr__", tp_repr, slot_tp_repr, wrap_unaryfunc, + "x.__repr__() <==> repr(x)"), + TPSLOT("__hash__", tp_hash, slot_tp_hash, wrap_hashfunc, + "x.__hash__() <==> hash(x)"), + FLSLOT("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call, + "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS), + TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc, + "x.__str__() <==> str(x)"), + TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook, + wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"), + TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""), + TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr, + "x.__setattr__('name', value) <==> x.name = value"), + TPSLOT("__delattr__", tp_setattro, slot_tp_setattro, wrap_delattr, + "x.__delattr__('name') <==> del x.name"), + TPSLOT("__lt__", tp_richcompare, slot_tp_richcompare, richcmp_lt, + "x.__lt__(y) <==> x x<=y"), + TPSLOT("__eq__", tp_richcompare, slot_tp_richcompare, richcmp_eq, + "x.__eq__(y) <==> x==y"), + TPSLOT("__ne__", tp_richcompare, slot_tp_richcompare, richcmp_ne, + "x.__ne__(y) <==> x!=y"), + TPSLOT("__gt__", tp_richcompare, slot_tp_richcompare, richcmp_gt, + "x.__gt__(y) <==> x>y"), + TPSLOT("__ge__", tp_richcompare, slot_tp_richcompare, richcmp_ge, + "x.__ge__(y) <==> x>=y"), + TPSLOT("__iter__", tp_iter, slot_tp_iter, wrap_unaryfunc, + "x.__iter__() <==> iter(x)"), + TPSLOT("next", tp_iternext, slot_tp_iternext, wrap_next, + "x.next() -> the next value, or raise StopIteration"), + TPSLOT("__get__", tp_descr_get, slot_tp_descr_get, wrap_descr_get, + "descr.__get__(obj[, type]) -> value"), + TPSLOT("__set__", tp_descr_set, slot_tp_descr_set, wrap_descr_set, + "descr.__set__(obj, value)"), + TPSLOT("__delete__", tp_descr_set, slot_tp_descr_set, + wrap_descr_delete, "descr.__delete__(obj)"), + FLSLOT("__init__", tp_init, slot_tp_init, (wrapperfunc)wrap_init, + "x.__init__(...) initializes x; " + "see help(type(x)) for signature", + PyWrapperFlag_KEYWORDS), + TPSLOT("__new__", tp_new, slot_tp_new, NULL, ""), + TPSLOT("__del__", tp_del, slot_tp_del, NULL, ""), BINSLOT("__add__", nb_add, slot_nb_add, "+"), RBINSLOT("__radd__", nb_add, slot_nb_add, @@ -5961,8 +5962,6 @@ static slotdef slotdefs[] = { "oct(x)"), UNSLOT("__hex__", nb_hex, slot_nb_hex, wrap_unaryfunc, "hex(x)"), - NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc, - "x[y:z] <==> x[y.__index__():z.__index__()]"), IBSLOT("__iadd__", nb_inplace_add, slot_nb_inplace_add, wrap_binaryfunc, "+="), IBSLOT("__isub__", nb_inplace_subtract, slot_nb_inplace_subtract, @@ -5993,58 +5992,57 @@ static slotdef slotdefs[] = { slot_nb_inplace_floor_divide, wrap_binaryfunc, "//"), IBSLOT("__itruediv__", nb_inplace_true_divide, slot_nb_inplace_true_divide, wrap_binaryfunc, "/"), - - TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc, - "x.__str__() <==> str(x)"), - TPSLOT("__str__", tp_print, NULL, NULL, ""), - TPSLOT("__repr__", tp_repr, slot_tp_repr, wrap_unaryfunc, - "x.__repr__() <==> repr(x)"), - TPSLOT("__repr__", tp_print, NULL, NULL, ""), - TPSLOT("__cmp__", tp_compare, _PyObject_SlotCompare, wrap_cmpfunc, - "x.__cmp__(y) <==> cmp(x,y)"), - TPSLOT("__hash__", tp_hash, slot_tp_hash, wrap_hashfunc, - "x.__hash__() <==> hash(x)"), - FLSLOT("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call, - "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS), - TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook, - wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"), - TPSLOT("__getattribute__", tp_getattr, NULL, NULL, ""), - TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""), - TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""), - TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr, - "x.__setattr__('name', value) <==> x.name = value"), - TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""), - TPSLOT("__delattr__", tp_setattro, slot_tp_setattro, wrap_delattr, - "x.__delattr__('name') <==> del x.name"), - TPSLOT("__delattr__", tp_setattr, NULL, NULL, ""), - TPSLOT("__lt__", tp_richcompare, slot_tp_richcompare, richcmp_lt, - "x.__lt__(y) <==> x x<=y"), - TPSLOT("__eq__", tp_richcompare, slot_tp_richcompare, richcmp_eq, - "x.__eq__(y) <==> x==y"), - TPSLOT("__ne__", tp_richcompare, slot_tp_richcompare, richcmp_ne, - "x.__ne__(y) <==> x!=y"), - TPSLOT("__gt__", tp_richcompare, slot_tp_richcompare, richcmp_gt, - "x.__gt__(y) <==> x>y"), - TPSLOT("__ge__", tp_richcompare, slot_tp_richcompare, richcmp_ge, - "x.__ge__(y) <==> x>=y"), - TPSLOT("__iter__", tp_iter, slot_tp_iter, wrap_unaryfunc, - "x.__iter__() <==> iter(x)"), - TPSLOT("next", tp_iternext, slot_tp_iternext, wrap_next, - "x.next() -> the next value, or raise StopIteration"), - TPSLOT("__get__", tp_descr_get, slot_tp_descr_get, wrap_descr_get, - "descr.__get__(obj[, type]) -> value"), - TPSLOT("__set__", tp_descr_set, slot_tp_descr_set, wrap_descr_set, - "descr.__set__(obj, value)"), - TPSLOT("__delete__", tp_descr_set, slot_tp_descr_set, - wrap_descr_delete, "descr.__delete__(obj)"), - FLSLOT("__init__", tp_init, slot_tp_init, (wrapperfunc)wrap_init, - "x.__init__(...) initializes x; " - "see help(type(x)) for signature", - PyWrapperFlag_KEYWORDS), - TPSLOT("__new__", tp_new, slot_tp_new, NULL, ""), - TPSLOT("__del__", tp_del, slot_tp_del, NULL, ""), + NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc, + "x[y:z] <==> x[y.__index__():z.__index__()]"), + MPSLOT("__len__", mp_length, slot_mp_length, wrap_lenfunc, + "x.__len__() <==> len(x)"), + MPSLOT("__getitem__", mp_subscript, slot_mp_subscript, + wrap_binaryfunc, + "x.__getitem__(y) <==> x[y]"), + MPSLOT("__setitem__", mp_ass_subscript, slot_mp_ass_subscript, + wrap_objobjargproc, + "x.__setitem__(i, y) <==> x[i]=y"), + MPSLOT("__delitem__", mp_ass_subscript, slot_mp_ass_subscript, + wrap_delitem, + "x.__delitem__(y) <==> del x[y]"), + SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc, + "x.__len__() <==> len(x)"), + /* Heap types defining __add__/__mul__ have sq_concat/sq_repeat == NULL. + The logic in abstract.c always falls back to nb_add/nb_multiply in + this case. Defining both the nb_* and the sq_* slots to call the + user-defined methods has unexpected side-effects, as shown by + test_descr.notimplemented() */ + SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc, + "x.__add__(y) <==> x+y"), + SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc, + "x.__mul__(n) <==> x*n"), + SQSLOT("__rmul__", sq_repeat, NULL, wrap_indexargfunc, + "x.__rmul__(n) <==> n*x"), + SQSLOT("__getitem__", sq_item, slot_sq_item, wrap_sq_item, + "x.__getitem__(y) <==> x[y]"), + SQSLOT("__getslice__", sq_slice, slot_sq_slice, wrap_ssizessizeargfunc, + "x.__getslice__(i, j) <==> x[i:j]\n\ + \n\ + Use of negative indices is not supported."), + SQSLOT("__setitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_setitem, + "x.__setitem__(i, y) <==> x[i]=y"), + SQSLOT("__delitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_delitem, + "x.__delitem__(y) <==> del x[y]"), + SQSLOT("__setslice__", sq_ass_slice, slot_sq_ass_slice, + wrap_ssizessizeobjargproc, + "x.__setslice__(i, j, y) <==> x[i:j]=y\n\ + \n\ + Use of negative indices is not supported."), + SQSLOT("__delslice__", sq_ass_slice, slot_sq_ass_slice, wrap_delslice, + "x.__delslice__(i, j) <==> del x[i:j]\n\ + \n\ + Use of negative indices is not supported."), + SQSLOT("__contains__", sq_contains, slot_sq_contains, wrap_objobjproc, + "x.__contains__(y) <==> y in x"), + SQSLOT("__iadd__", sq_inplace_concat, NULL, + wrap_binaryfunc, "x.__iadd__(y) <==> x+=y"), + SQSLOT("__imul__", sq_inplace_repeat, NULL, + wrap_indexargfunc, "x.__imul__(y) <==> x*=y"), {NULL} }; @@ -6225,21 +6223,6 @@ update_slots_callback(PyTypeObject *type, void *data) return 0; } -/* Comparison function for qsort() to compare slotdefs by their offset, and - for equal offset by their address (to force a stable sort). */ -static int -slotdef_cmp(const void *aa, const void *bb) -{ - const slotdef *a = (const slotdef *)aa, *b = (const slotdef *)bb; - int c = a->offset - b->offset; - if (c != 0) - return c; - else - /* Cannot use a-b, as this gives off_t, - which may lose precision when converted to int. */ - return (a > b) ? 1 : (a < b) ? -1 : 0; -} - /* Initialize the slotdefs table by adding interned string objects for the names and sorting the entries. */ static void @@ -6251,12 +6234,12 @@ init_slotdefs(void) if (initialized) return; for (p = slotdefs; p->name; p++) { + /* Slots must be ordered by their offset in the PyHeapTypeObject. */ + assert(!p[1].name || p->offset <= p[1].offset); p->name_strobj = PyString_InternFromString(p->name); if (!p->name_strobj) Py_FatalError("Out of memory interning slotdef names"); } - qsort((void *)slotdefs, (size_t)(p-slotdefs), sizeof(slotdef), - slotdef_cmp); initialized = 1; } From b37a554779a076f6ab5a6ba63ef0d9c3d93bdea7 Mon Sep 17 00:00:00 2001 From: Roger Serwy Date: Sun, 7 Apr 2013 12:15:52 -0500 Subject: [PATCH 024/480] #1207589: Backwards-compatibility patch for right-click menu in IDLE. --- Lib/idlelib/EditorWindow.py | 10 ++++++++-- Misc/NEWS | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/EditorWindow.py b/Lib/idlelib/EditorWindow.py index 96f1ee58784..cf3fad3c1ec 100644 --- a/Lib/idlelib/EditorWindow.py +++ b/Lib/idlelib/EditorWindow.py @@ -480,7 +480,12 @@ def right_menu_event(self, event): if iswin: self.text.config(cursor="arrow") - for label, eventname, verify_state in self.rmenu_specs: + for item in self.rmenu_specs: + try: + label, eventname, verify_state = item + except ValueError: # see issue1207589 + continue + if verify_state is None: continue state = getattr(self, verify_state)() @@ -497,7 +502,8 @@ def right_menu_event(self, event): def make_rmenu(self): rmenu = Menu(self.text, tearoff=0) - for label, eventname, _ in self.rmenu_specs: + for item in self.rmenu_specs: + label, eventname = item[0], item[1] if label is not None: def command(text=self.text, eventname=eventname): text.event_generate(eventname) diff --git a/Misc/NEWS b/Misc/NEWS index 3d9e9dcb581..abb5068e3dc 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -23,6 +23,8 @@ Library IDLE ---- +- Issue #1207589: Backwards-compatibility patch for right-click menu in IDLE. + - Issue #16887: IDLE now accepts Cancel in tabify/untabify dialog box. - Issue #14254: IDLE now handles readline correctly across shell restarts. From 9cbb40c2cd91014bbf9136f8d909f17177dab64b Mon Sep 17 00:00:00 2001 From: Roger Serwy Date: Sun, 7 Apr 2013 12:41:16 -0500 Subject: [PATCH 025/480] #17613: Prevent traceback when removing syntax colorizer in IDLE. --- Lib/idlelib/ColorDelegator.py | 4 ++++ Misc/NEWS | 2 ++ 2 files changed, 6 insertions(+) diff --git a/Lib/idlelib/ColorDelegator.py b/Lib/idlelib/ColorDelegator.py index 0610c4b6aac..c74865d4fb4 100644 --- a/Lib/idlelib/ColorDelegator.py +++ b/Lib/idlelib/ColorDelegator.py @@ -50,6 +50,10 @@ def setdelegate(self, delegate): self.config_colors() self.bind("<>", self.toggle_colorize_event) self.notify_range("1.0", "end") + else: + # No delegate - stop any colorizing + self.stop_colorizing = True + self.allow_colorizing = False def config_colors(self): for tag, cnf in self.tagdefs.items(): diff --git a/Misc/NEWS b/Misc/NEWS index abb5068e3dc..e2ed81c289e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -23,6 +23,8 @@ Library IDLE ---- +- Issue #17613: Prevent traceback when removing syntax colorizer in IDLE. + - Issue #1207589: Backwards-compatibility patch for right-click menu in IDLE. - Issue #16887: IDLE now accepts Cancel in tabify/untabify dialog box. From 63761735d1e902d5badc12769dcf3ac415384733 Mon Sep 17 00:00:00 2001 From: Kristjan Valur Jonsson Date: Mon, 8 Apr 2013 16:03:30 +0000 Subject: [PATCH 026/480] Issue #17662: fix socketmodule compilation with Setup.dist. Patch contributed by Bohuslav "Slavek" Kabrda (bkabrda) --- Modules/Setup.dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/Setup.dist b/Modules/Setup.dist index 1121ae64a84..289d4b4ef84 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -208,7 +208,7 @@ GLHACK=-Dclear=__GLclear #_csv _csv.c # Socket module helper for socket(2) -#_socket socketmodule.c +#_socket socketmodule.c timemodule.c # Socket module helper for SSL support; you must comment out the other # socket line above, and possibly edit the SSL variable: From 8fd1837830be5b8481bf39cb4155804c4c8816b3 Mon Sep 17 00:00:00 2001 From: doko Date: Mon, 8 Apr 2013 21:20:09 +0200 Subject: [PATCH 027/480] - Issue #13150, #17512: sysconfig no longer parses the Makefile and config.h files when imported, instead doing it at build time. This makes importing sysconfig faster and reduces Python startup time by 20%. --- .bzrignore | 1 + .hgignore | 1 + Lib/distutils/sysconfig.py | 63 ++------------------------------ Lib/pprint.py | 5 ++- Lib/site.py | 15 -------- Lib/sysconfig.py | 75 ++++++++++++++++++++++++++++++++++++-- Makefile.pre.in | 13 +++++-- Misc/NEWS | 4 ++ Modules/getpath.c | 23 ++++++++++-- 9 files changed, 115 insertions(+), 85 deletions(-) diff --git a/.bzrignore b/.bzrignore index 2527052a1ba..abca6b1c5d6 100644 --- a/.bzrignore +++ b/.bzrignore @@ -11,6 +11,7 @@ python build Makefile.pre platform +pybuilddir.txt pyconfig.h libpython*.a libpython*.so* diff --git a/.hgignore b/.hgignore index de9bd5dbe29..c51c53cfbd7 100644 --- a/.hgignore +++ b/.hgignore @@ -35,6 +35,7 @@ Parser/pgen$ Parser/pgen.stamp$ ^core ^python-gdb.py +^pybuilddir.txt syntax: glob python.exe-gdb.py diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 250ef38bedf..0c726d92ffb 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -387,66 +387,11 @@ def expand_makefile_vars(s, vars): def _init_posix(): """Initialize the module as appropriate for POSIX systems.""" - g = {} - # load the installed Makefile: - try: - filename = get_makefile_filename() - parse_makefile(filename, g) - except IOError, msg: - my_msg = "invalid Python installation: unable to open %s" % filename - if hasattr(msg, "strerror"): - my_msg = my_msg + " (%s)" % msg.strerror - - raise DistutilsPlatformError(my_msg) - - # load the installed pyconfig.h: - try: - filename = get_config_h_filename() - parse_config_h(file(filename), g) - except IOError, msg: - my_msg = "invalid Python installation: unable to open %s" % filename - if hasattr(msg, "strerror"): - my_msg = my_msg + " (%s)" % msg.strerror - - raise DistutilsPlatformError(my_msg) - - # On AIX, there are wrong paths to the linker scripts in the Makefile - # -- these paths are relative to the Python source, but when installed - # the scripts are in another directory. - if python_build: - g['LDSHARED'] = g['BLDSHARED'] - - elif get_python_version() < '2.1': - # The following two branches are for 1.5.2 compatibility. - if sys.platform == 'aix4': # what about AIX 3.x ? - # Linker script is in the config directory, not in Modules as the - # Makefile says. - python_lib = get_python_lib(standard_lib=1) - ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix') - python_exp = os.path.join(python_lib, 'config', 'python.exp') - - g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp) - - elif sys.platform == 'beos': - # Linker script is in the config directory. In the Makefile it is - # relative to the srcdir, which after installation no longer makes - # sense. - python_lib = get_python_lib(standard_lib=1) - linkerscript_path = string.split(g['LDSHARED'])[0] - linkerscript_name = os.path.basename(linkerscript_path) - linkerscript = os.path.join(python_lib, 'config', - linkerscript_name) - - # XXX this isn't the right place to do this: adding the Python - # library to the link, if needed, should be in the "build_ext" - # command. (It's also needed for non-MS compilers on Windows, and - # it's taken care of for them by the 'build_ext.get_libraries()' - # method.) - g['LDSHARED'] = ("%s -L%s/lib -lpython%s" % - (linkerscript, PREFIX, get_python_version())) - + # _sysconfigdata is generated at build time, see the sysconfig module + from _sysconfigdata import build_time_vars global _config_vars - _config_vars = g + _config_vars = {} + _config_vars.update(build_time_vars) def _init_nt(): diff --git a/Lib/pprint.py b/Lib/pprint.py index 910283e6091..330099dfea6 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -37,7 +37,10 @@ import sys as _sys import warnings -from cStringIO import StringIO as _StringIO +try: + from cStringIO import StringIO as _StringIO +except ImportError: + from StringIO import StringIO as _StringIO __all__ = ["pprint","pformat","isreadable","isrecursive","saferepr", "PrettyPrinter"] diff --git a/Lib/site.py b/Lib/site.py index 54e515479a9..f1b0ae8c645 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -114,18 +114,6 @@ def removeduppaths(): sys.path[:] = L return known_paths -# XXX This should not be part of site.py, since it is needed even when -# using the -S option for Python. See http://www.python.org/sf/586680 -def addbuilddir(): - """Append ./build/lib. in case we're running in the build dir - (especially for Guido :-)""" - from sysconfig import get_platform - s = "build/lib.%s-%.3s" % (get_platform(), sys.version) - if hasattr(sys, 'gettotalrefcount'): - s += '-pydebug' - s = os.path.join(os.path.dirname(sys.path.pop()), s) - sys.path.append(s) - def _init_pathinfo(): """Return a set containing all existing directory entries from sys.path""" @@ -537,9 +525,6 @@ def main(): abs__file__() known_paths = removeduppaths() - if (os.name == "posix" and sys.path and - os.path.basename(sys.path[-1]) == "Modules"): - addbuilddir() if ENABLE_USER_SITE is None: ENABLE_USER_SITE = check_enableusersite() known_paths = addusersitepackages(known_paths) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index d74ca391c64..aa69351bf0e 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -278,9 +278,10 @@ def _get_makefile_filename(): return os.path.join(_PROJECT_BASE, "Makefile") return os.path.join(get_path('platstdlib'), "config", "Makefile") - -def _init_posix(vars): - """Initialize the module as appropriate for POSIX systems.""" +def _generate_posix_vars(): + """Generate the Python module containing build-time variables.""" + import pprint + vars = {} # load the installed Makefile: makefile = _get_makefile_filename() try: @@ -308,6 +309,49 @@ def _init_posix(vars): if _PYTHON_BUILD: vars['LDSHARED'] = vars['BLDSHARED'] + # There's a chicken-and-egg situation on OS X with regards to the + # _sysconfigdata module after the changes introduced by #15298: + # get_config_vars() is called by get_platform() as part of the + # `make pybuilddir.txt` target -- which is a precursor to the + # _sysconfigdata.py module being constructed. Unfortunately, + # get_config_vars() eventually calls _init_posix(), which attempts + # to import _sysconfigdata, which we won't have built yet. In order + # for _init_posix() to work, if we're on Darwin, just mock up the + # _sysconfigdata module manually and populate it with the build vars. + # This is more than sufficient for ensuring the subsequent call to + # get_platform() succeeds. + name = '_sysconfigdata' + if 'darwin' in sys.platform: + import imp + module = imp.new_module(name) + module.build_time_vars = vars + sys.modules[name] = module + + pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version[:3]) + if hasattr(sys, "gettotalrefcount"): + pybuilddir += '-pydebug' + try: + os.makedirs(pybuilddir) + except OSError: + pass + destfile = os.path.join(pybuilddir, name + '.py') + + with open(destfile, 'wb') as f: + f.write('# system configuration generated and used by' + ' the sysconfig module\n') + f.write('build_time_vars = ') + pprint.pprint(vars, stream=f) + + # Create file used for sys.path fixup -- see Modules/getpath.c + with open('pybuilddir.txt', 'w') as f: + f.write(pybuilddir) + +def _init_posix(vars): + """Initialize the module as appropriate for POSIX systems.""" + # _sysconfigdata is generated at build time, see _generate_posix_vars() + from _sysconfigdata import build_time_vars + vars.update(build_time_vars) + def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories @@ -565,3 +609,28 @@ def get_platform(): def get_python_version(): return _PY_VERSION_SHORT + + +def _print_dict(title, data): + for index, (key, value) in enumerate(sorted(data.items())): + if index == 0: + print '%s: ' % (title) + print '\t%s = "%s"' % (key, value) + + +def _main(): + """Display all information sysconfig detains.""" + if '--generate-posix-vars' in sys.argv: + _generate_posix_vars() + return + print 'Platform: "%s"' % get_platform() + print 'Python version: "%s"' % get_python_version() + print 'Current installation scheme: "%s"' % _get_default_scheme() + print + _print_dict('Paths', get_paths()) + print + _print_dict('Variables', get_config_vars()) + + +if __name__ == '__main__': + _main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 93fb0cf40c8..9d555503c14 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -437,15 +437,20 @@ $(BUILDPYTHON): Modules/python.o $(LIBRARY) $(LDLIBRARY) Modules/python.o \ $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) -platform: $(BUILDPYTHON) +platform: $(BUILDPYTHON) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print get_platform()+"-"+sys.version[0:3]' >platform +# Create build directory and generate the sysconfig build-time data there. +# pybuilddir.txt contains the name of the build dir and is used for +# sys.path fixup -- see Modules/getpath.c. +pybuilddir.txt: $(BUILDPYTHON) + $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars # Build the shared modules # Under GNU make, MAKEFLAGS are sorted and normalized; the 's' for # -s, --silent or --quiet is always the first char. # Under BSD make, MAKEFLAGS might be " -s -v x=y". -sharedmods: $(BUILDPYTHON) +sharedmods: $(BUILDPYTHON) pybuilddir.txt @case "$$MAKEFLAGS" in \ *\ -s*|s*) quiet="-q";; \ *) quiet="";; \ @@ -955,7 +960,7 @@ libinstall: build_all $(srcdir)/Lib/$(PLATDIR) $(srcdir)/Modules/xxmodule.c else true; \ fi; \ done - @for i in $(srcdir)/Lib/*.py $(srcdir)/Lib/*.doc $(srcdir)/Lib/*.egg-info ; \ + @for i in $(srcdir)/Lib/*.py `cat pybuilddir.txt`/_sysconfigdata.py $(srcdir)/Lib/*.doc $(srcdir)/Lib/*.egg-info ; \ do \ if test -x $$i; then \ $(INSTALL_SCRIPT) $$i $(DESTDIR)$(LIBDEST); \ @@ -1133,6 +1138,7 @@ sharedinstall: sharedmods --install-scripts=$(BINDIR) \ --install-platlib=$(DESTSHARED) \ --root=$(DESTDIR)/ + -rm $(DESTDIR)$(DESTSHARED)/_sysconfigdata.py* # Here are a couple of targets for MacOSX again, to install a full # framework-based Python. frameworkinstall installs everything, the @@ -1288,6 +1294,7 @@ distclean: clobber Modules/Setup Modules/Setup.local Modules/Setup.config \ Modules/ld_so_aix Modules/python.exp Misc/python.pc -rm -f python*-gdb.py + -rm -f pybuilddir.txt find $(srcdir) '(' -name '*.fdc' -o -name '*~' \ -o -name '[@,#]*' -o -name '*.old' \ -o -name '*.orig' -o -name '*.rej' \ diff --git a/Misc/NEWS b/Misc/NEWS index e2ed81c289e..57e0cc6575f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -14,6 +14,10 @@ Core and Builtins Library ------- +- Issue #13150, #17512: sysconfig no longer parses the Makefile and config.h + files when imported, instead doing it at build time. This makes importing + sysconfig faster and reduces Python startup time by 20%. + - Issue #13163: Rename operands in smtplib.SMTP._get_socket to correct names; fixes otherwise misleading output in tracebacks and when when debug is on. diff --git a/Modules/getpath.c b/Modules/getpath.c index 9faafa3555d..de96d474d21 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -335,12 +335,27 @@ search_for_exec_prefix(char *argv0_path, char *home) return 1; } - /* Check to see if argv[0] is in the build directory */ + /* Check to see if argv[0] is in the build directory. "pybuilddir.txt" + is written by setup.py and contains the relative path to the location + of shared library modules. */ strcpy(exec_prefix, argv0_path); - joinpath(exec_prefix, "Modules/Setup"); + joinpath(exec_prefix, "pybuilddir.txt"); if (isfile(exec_prefix)) { - reduce(exec_prefix); - return -1; + FILE *f = fopen(exec_prefix, "r"); + if (f == NULL) + errno = 0; + else { + char rel_builddir_path[MAXPATHLEN+1]; + size_t n; + n = fread(rel_builddir_path, 1, MAXPATHLEN, f); + rel_builddir_path[n] = '\0'; + fclose(f); + if (n >= 0) { + strcpy(exec_prefix, argv0_path); + joinpath(exec_prefix, rel_builddir_path); + return -1; + } + } } /* Search from argv0_path, until root is found */ From d2a2cf386e79fbc6dbb764a6cd3426125e28f0b9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 8 Apr 2013 22:33:55 +0300 Subject: [PATCH 028/480] Close #17666: Fix reading gzip files with an extra field. --- Lib/gzip.py | 3 ++- Lib/test/test_gzip.py | 7 +++++++ Misc/NEWS | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Lib/gzip.py b/Lib/gzip.py index a2f23679fa8..99b2d3ef078 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -202,7 +202,8 @@ def _read_gzip_header(self): if flag & FEXTRA: # Read & discard the extra field, if present - self._read_exact(struct.unpack(" Date: Mon, 8 Apr 2013 20:57:13 -0500 Subject: [PATCH 029/480] #17657: Show full Tk version in IDLE's about dialog. Patch by Todd Rovito. --- Lib/idlelib/aboutDialog.py | 7 +------ Misc/NEWS | 3 +++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Lib/idlelib/aboutDialog.py b/Lib/idlelib/aboutDialog.py index 43a13135aee..07345180f91 100644 --- a/Lib/idlelib/aboutDialog.py +++ b/Lib/idlelib/aboutDialog.py @@ -66,12 +66,7 @@ def CreateWidgets(self): labelPythonVer = Label(frameBg, text='Python version: ' + \ sys.version.split()[0], fg=self.fg, bg=self.bg) labelPythonVer.grid(row=9, column=0, sticky=W, padx=10, pady=0) - # handle weird tk version num in windoze python >= 1.6 (?!?) - tkVer = repr(TkVersion).split('.') - tkVer[len(tkVer)-1] = str('%.3g' % (float('.'+tkVer[len(tkVer)-1])))[2:] - if tkVer[len(tkVer)-1] == '': - tkVer[len(tkVer)-1] = '0' - tkVer = '.'.join(tkVer) + tkVer = self.tk.call('info', 'patchlevel') labelTkVer = Label(frameBg, text='Tk version: '+ tkVer, fg=self.fg, bg=self.bg) labelTkVer.grid(row=9, column=1, sticky=W, padx=2, pady=0) diff --git a/Misc/NEWS b/Misc/NEWS index deb0d4e2bae..161776daa04 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,9 @@ Library IDLE ---- +- Issue #17657: Show full Tk version in IDLE's about dialog. + Patch by Todd Rovito. + - Issue #17613: Prevent traceback when removing syntax colorizer in IDLE. - Issue #1207589: Backwards-compatibility patch for right-click menu in IDLE. From 7470f3e655e82a4565ce55476ba180b7fc18da9b Mon Sep 17 00:00:00 2001 From: doko Date: Tue, 9 Apr 2013 17:34:39 +0200 Subject: [PATCH 030/480] - Issue #17682: Add the _io module to Modules/Setup.dist (commented out). --- Misc/NEWS | 5 +++++ Modules/Setup.dist | 3 +++ 2 files changed, 8 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 161776daa04..b6f15f33b37 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -6,6 +6,11 @@ What's New in Python 2.7.5? *Release date: XXXX-XX-XX* +Build +----- + +- Issue #17682: Add the _io module to Modules/Setup.dist (commented out). + Core and Builtins ----------------- diff --git a/Modules/Setup.dist b/Modules/Setup.dist index 289d4b4ef84..2ad1aa33d33 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -191,6 +191,9 @@ GLHACK=-Dclear=__GLclear # access to ISO C locale support #_locale _localemodule.c # -lintl +# Standard I/O baseline +#_io -I$(srcdir)/Modules/_io _io/bufferedio.c _io/bytesio.c _io/fileio.c _io/iobase.c _io/_iomodule.c _io/stringio.c _io/textio.c + # Modules with some UNIX dependencies -- on by default: # (If you have a really backward UNIX, select and socket may not be From e3ce7f1dfa71538b6cec0afc5830168f8d6d203f Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Wed, 10 Apr 2013 17:59:20 +0300 Subject: [PATCH 031/480] #17635: fix wrong function name in multiprocessing docs. --- Doc/library/multiprocessing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index b7d11941c73..6f9298ad28d 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1744,7 +1744,7 @@ authentication* using the :mod:`hmac` module. then a welcome message is sent to the other end of the connection. Otherwise :exc:`AuthenticationError` is raised. -.. function:: answerChallenge(connection, authkey) +.. function:: answer_challenge(connection, authkey) Receive a message, calculate the digest of the message using *authkey* as the key, and then send the digest back. From 8e6479759e0af51ccf6985a84208fbbc86e30caf Mon Sep 17 00:00:00 2001 From: doko Date: Thu, 11 Apr 2013 00:19:55 +0200 Subject: [PATCH 032/480] - Issue #17086: Search the include and library directories provided by the compiler. --- Misc/NEWS | 3 +++ setup.py | 1 + 2 files changed, 4 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index b6f15f33b37..76718bedd92 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -11,6 +11,9 @@ Build - Issue #17682: Add the _io module to Modules/Setup.dist (commented out). +- Issue #17086: Search the include and library directories provided by the + compiler. + Core and Builtins ----------------- diff --git a/setup.py b/setup.py index ea8a5f51e96..716f08e2112 100644 --- a/setup.py +++ b/setup.py @@ -439,6 +439,7 @@ def detect_modules(self): # Ensure that /usr/local is always used add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib') add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') + self.add_gcc_paths() self.add_multiarch_paths() # Add paths specified in the environment variables LDFLAGS and From fdbad511644b96636a81652ff2626243bfcbefcb Mon Sep 17 00:00:00 2001 From: Roger Serwy Date: Thu, 11 Apr 2013 19:13:21 -0500 Subject: [PATCH 033/480] #17585: Fixed IDLE regression. Now closes when using exit() or quit(). --- Lib/idlelib/PyShell.py | 3 +++ Misc/NEWS | 2 ++ 2 files changed, 5 insertions(+) diff --git a/Lib/idlelib/PyShell.py b/Lib/idlelib/PyShell.py index 9451d2e6c92..81af85a8404 100644 --- a/Lib/idlelib/PyShell.py +++ b/Lib/idlelib/PyShell.py @@ -1370,6 +1370,9 @@ def readline(self, size=-1): self._line_buffer = line[size:] return line[:size] + def close(self): + self.shell.close() + usage_msg = """\ diff --git a/Misc/NEWS b/Misc/NEWS index 76718bedd92..a04210c21b5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -37,6 +37,8 @@ Library IDLE ---- +- Issue #17585: Fixed IDLE regression. Now closes when using exit() or quit(). + - Issue #17657: Show full Tk version in IDLE's about dialog. Patch by Todd Rovito. From 0ecd4c0f754b77b8a88ee4d42e54707c6d0f23d5 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Fri, 12 Apr 2013 16:22:24 +0300 Subject: [PATCH 034/480] #6696: add documentation for the Profile objects, and improve profile/cProfile docs. Patch by Tom Pinckney. --- Doc/library/profile.rst | 774 +++++++++++++++++++++------------------- Misc/ACKS | 1 + Misc/NEWS | 3 + 3 files changed, 404 insertions(+), 374 deletions(-) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 627f2156ab9..e2757f6632a 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -4,11 +4,6 @@ The Python Profilers ******************** -.. sectionauthor:: James Roskind - -.. module:: profile - :synopsis: Python source profiler. - **Source code:** :source:`Lib/profile.py` and :source:`Lib/pstats.py` -------------- @@ -22,34 +17,31 @@ Introduction to the profilers single: deterministic profiling single: profiling, deterministic -A :dfn:`profiler` is a program that describes the run time performance -of a program, providing a variety of statistics. This documentation -describes the profiler functionality provided in the modules -:mod:`cProfile`, :mod:`profile` and :mod:`pstats`. This profiler -provides :dfn:`deterministic profiling` of Python programs. It also -provides a series of report generation tools to allow users to rapidly -examine the results of a profile operation. +:mod:`cProfile` and :mod:`profile` provide :dfn:`deterministic profiling` of +Python programs. A :dfn:`profile` is a set of statistics that describes how +often and for how long various parts of the program executed. These statistics +can be formatted into reports via the :mod:`pstats` module. -The Python standard library provides three different profilers: +The Python standard library provides three different implementations of the same +profiling interface: -#. :mod:`cProfile` is recommended for most users; it's a C extension - with reasonable overhead - that makes it suitable for profiling long-running programs. - Based on :mod:`lsprof`, - contributed by Brett Rosen and Ted Czotter. +1. :mod:`cProfile` is recommended for most users; it's a C extension with + reasonable overhead that makes it suitable for profiling long-running + programs. Based on :mod:`lsprof`, contributed by Brett Rosen and Ted + Czotter. .. versionadded:: 2.5 -#. :mod:`profile`, a pure Python module whose interface is imitated by - :mod:`cProfile`. Adds significant overhead to profiled programs. - If you're trying to extend - the profiler in some way, the task might be easier with this module. +2. :mod:`profile`, a pure Python module whose interface is imitated by + :mod:`cProfile`, but which adds significant overhead to profiled programs. + If you're trying to extend the profiler in some way, the task might be easier + with this module. .. versionchanged:: 2.4 Now also reports the time spent in calls to built-in functions and methods. -#. :mod:`hotshot` was an experimental C module that focused on minimizing +3. :mod:`hotshot` was an experimental C module that focused on minimizing the overhead of profiling, at the expense of longer data post-processing times. It is no longer maintained and may be dropped in a future version of Python. @@ -66,6 +58,15 @@ is newer and might not be available on all systems. :mod:`_lsprof` module. The :mod:`hotshot` module is reserved for specialized usage. +.. note:: + + The profiler modules are designed to provide an execution profile for a given + program, not for benchmarking purposes (for that, there is :mod:`timeit` for + reasonably accurate results). This particularly applies to benchmarking + Python code against C code: the profilers introduce overhead for Python code, + but not for C-level functions, and so the C code would seem faster than any + Python one. + .. _profile-instant: @@ -76,57 +77,94 @@ This section is provided for users that "don't want to read the manual." It provides a very brief overview, and allows a user to rapidly perform profiling on an existing application. -To profile an application with a main entry point of :func:`foo`, you would add -the following to your module:: +To profile a function that takes a single argument, you can do:: import cProfile - cProfile.run('foo()') + import re + cProfile.run('re.compile("foo|bar")') (Use :mod:`profile` instead of :mod:`cProfile` if the latter is not available on your system.) -The above action would cause :func:`foo` to be run, and a series of informative -lines (the profile) to be printed. The above approach is most useful when -working with the interpreter. If you would like to save the results of a -profile into a file for later examination, you can supply a file name as the -second argument to the :func:`run` function:: +The above action would run :func:`re.compile` and print profile results like +the following:: - import cProfile - cProfile.run('foo()', 'fooprof') + 197 function calls (192 primitive calls) in 0.002 seconds -The file :file:`cProfile.py` can also be invoked as a script to profile another -script. For example:: + Ordered by: standard name - python -m cProfile myscript.py + ncalls tottime percall cumtime percall filename:lineno(function) + 1 0.000 0.000 0.001 0.001 :1() + 1 0.000 0.000 0.001 0.001 re.py:212(compile) + 1 0.000 0.000 0.001 0.001 re.py:268(_compile) + 1 0.000 0.000 0.000 0.000 sre_compile.py:172(_compile_charset) + 1 0.000 0.000 0.000 0.000 sre_compile.py:201(_optimize_charset) + 4 0.000 0.000 0.000 0.000 sre_compile.py:25(_identityfunction) + 3/1 0.000 0.000 0.000 0.000 sre_compile.py:33(_compile) -:file:`cProfile.py` accepts two optional arguments on the command line:: +The first line indicates that 197 calls were monitored. Of those calls, 192 +were :dfn:`primitive`, meaning that the call was not induced via recursion. The +next line: ``Ordered by: standard name``, indicates that the text string in the +far right column was used to sort the output. The column headings include: - cProfile.py [-o output_file] [-s sort_order] +ncalls + for the number of calls, -``-s`` only applies to standard output (``-o`` is not supplied). -Look in the :class:`Stats` documentation for valid sort values. +tottime + for the total time spent in the given function (and excluding time made in + calls to sub-functions) -When you wish to review the profile, you should use the methods in the -:mod:`pstats` module. Typically you would load the statistics data as follows:: +percall + is the quotient of ``tottime`` divided by ``ncalls`` - import pstats - p = pstats.Stats('fooprof') +cumtime + is the cumulative time spent in this and all subfunctions (from invocation + till exit). This figure is accurate *even* for recursive functions. -The class :class:`Stats` (the above code just created an instance of this class) -has a variety of methods for manipulating and printing the data that was just -read into ``p``. When you ran :func:`cProfile.run` above, what was printed was -the result of three method calls:: +percall + is the quotient of ``cumtime`` divided by primitive calls - p.strip_dirs().sort_stats(-1).print_stats() +filename:lineno(function) + provides the respective data of each function + +When there are two numbers in the first column (for example ``3/1``), it means +that the function recursed. The second value is the number of primitive calls +and the former is the total number of calls. Note that when the function does +not recurse, these two values are the same, and only the single figure is +printed. -The first method removed the extraneous path from all the module names. The -second method sorted all the entries according to the standard module/line/name -string that is printed. The third method printed out all the statistics. You -might try the following sort calls: +Instead of printing the output at the end of the profile run, you can save the +results to a file by specifying a filename to the :func:`run` function:: -.. (this is to comply with the semantics of the old profiler). + import cProfile + import re + cProfile.run('re.compile("foo|bar")', 'restats') + +The :class:`pstats.Stats` class reads profile results from a file and formats +them in various ways. + +The file :mod:`cProfile` can also be invoked as a script to profile another +script. For example:: + + python -m cProfile [-o output_file] [-s sort_order] myscript.py + +``-o`` writes the profile results to a file instead of to stdout -:: +``-s`` specifies one of the :func:`~pstats.Stats.sort_stats` sort values to sort +the output by. This only applies when ``-o`` is not supplied. + +The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods +for manipulating and printing the data saved into a profile results file:: + + import pstats + p = pstats.Stats('restats') + p.strip_dirs().sort_stats(-1).print_stats() + +The :meth:`~pstats.Stats.strip_dirs` method removed the extraneous path from all +the module names. The :meth:`~pstats.Stats.sort_stats` method sorted all the +entries according to the standard module/line/name string that is printed. The +:meth:`~pstats.Stats.print_stats` method printed out all the statistics. You +might try the following sort calls:: p.sort_stats('name') p.print_stats() @@ -175,343 +213,338 @@ If you want more functionality, you're going to have to read the manual, or guess what the following functions do:: p.print_callees() - p.add('fooprof') + p.add('restats') Invoked as a script, the :mod:`pstats` module is a statistics browser for reading and examining profile dumps. It has a simple line-oriented interface (implemented using :mod:`cmd`) and interactive help. +:mod:`profile` and :mod:`cProfile` Module Reference +======================================================= -.. _deterministic-profiling: +.. module:: cProfile +.. module:: profile + :synopsis: Python source profiler. -What Is Deterministic Profiling? -================================ +Both the :mod:`profile` and :mod:`cProfile` modules provide the following +functions: -:dfn:`Deterministic profiling` is meant to reflect the fact that all *function -call*, *function return*, and *exception* events are monitored, and precise -timings are made for the intervals between these events (during which time the -user's code is executing). In contrast, :dfn:`statistical profiling` (which is -not done by this module) randomly samples the effective instruction pointer, and -deduces where time is being spent. The latter technique traditionally involves -less overhead (as the code does not need to be instrumented), but provides only -relative indications of where time is being spent. +.. function:: run(command, filename=None, sort=-1) -In Python, since there is an interpreter active during execution, the presence -of instrumented code is not required to do deterministic profiling. Python -automatically provides a :dfn:`hook` (optional callback) for each event. In -addition, the interpreted nature of Python tends to add so much overhead to -execution, that deterministic profiling tends to only add small processing -overhead in typical applications. The result is that deterministic profiling is -not that expensive, yet provides extensive run time statistics about the -execution of a Python program. + This function takes a single argument that can be passed to the :func:`exec` + function, and an optional file name. In all cases this routine executes:: -Call count statistics can be used to identify bugs in code (surprising counts), -and to identify possible inline-expansion points (high call counts). Internal -time statistics can be used to identify "hot loops" that should be carefully -optimized. Cumulative time statistics should be used to identify high level -errors in the selection of algorithms. Note that the unusual handling of -cumulative times in this profiler allows statistics for recursive -implementations of algorithms to be directly compared to iterative -implementations. + exec(command, __main__.__dict__, __main__.__dict__) + and gathers profiling statistics from the execution. If no file name is + present, then this function automatically creates a :class:`~pstats.Stats` + instance and prints a simple profiling report. If the sort value is specified + it is passed to this :class:`~pstats.Stats` instance to control how the + results are sorted. -Reference Manual -- :mod:`profile` and :mod:`cProfile` -====================================================== +.. function:: runctx(command, globals, locals, filename=None) -.. module:: cProfile - :synopsis: Python profiler + This function is similar to :func:`run`, with added arguments to supply the + globals and locals dictionaries for the *command* string. This routine + executes:: + exec(command, globals, locals) -The primary entry point for the profiler is the global function -:func:`profile.run` (resp. :func:`cProfile.run`). It is typically used to create -any profile information. The reports are formatted and printed using methods of -the class :class:`pstats.Stats`. The following is a description of all of these -standard entry points and functions. For a more in-depth view of some of the -code, consider reading the later section on Profiler Extensions, which includes -discussion of how to derive "better" profilers from the classes presented, or -reading the source code for these modules. + and gathers profiling statistics as in the :func:`run` function above. +.. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True) -.. function:: run(command[, filename][, sort]) + This class is normally only used if more precise control over profiling is + needed than what the :func:`cProfile.run` function provides. - This function takes a single argument that can be passed to the - :keyword:`exec` statement, and optionally a file name and a sorting - directive. In all cases this routine attempts to :keyword:`exec` - its first argument, and gather profiling statistics from the - execution. If no file name is present, then this function - automatically prints a simple profiling report, sorted by the - standard name string (file/line/function-name) that is presented in - each line. The following is a typical output from such a call:: + A custom timer can be supplied for measuring how long code takes to run via + the *timer* argument. This must be a function that returns a single number + representing the current time. If the number is an integer, the *timeunit* + specifies a multiplier that specifies the duration of each unit of time. For + example, if the timer returns times measured in thousands of seconds, the + time unit would be ``.001``. - 2706 function calls (2004 primitive calls) in 4.504 CPU seconds + Directly using the :class:`Profile` class allows formatting profile results + without writing the profile data to a file:: - Ordered by: standard name + import cProfile, pstats, io + pr = cProfile.Profile() + pr.enable() + ... do something ... + pr.disable() + s = io.StringIO() + ps = pstats.Stats(pr, stream=s) + ps.print_results() - ncalls tottime percall cumtime percall filename:lineno(function) - 2 0.006 0.003 0.953 0.477 pobject.py:75(save_objects) - 43/3 0.533 0.012 0.749 0.250 pobject.py:99(evaluate) - ... + .. method:: enable() - The first line indicates that 2706 calls were monitored. Of those - calls, 2004 were :dfn:`primitive`. We define :dfn:`primitive` to - mean that the call was not induced via recursion. The next line: - ``Ordered by: standard name``, indicates that the text string in - the far right column was used to sort the output. The column - headings include: + Start collecting profiling data. - ncalls - for the number of calls, + .. method:: disable() - tottime - for the total time spent in the given function (and - excluding time made in calls to sub-functions), + Stop collecting profiling data. - percall - is the quotient of ``tottime`` divided by ``ncalls`` + .. method:: create_stats() - cumtime - is the total time spent in this and all subfunctions (from invocation till - exit). This figure is accurate *even* for recursive functions. + Stop collecting profiling data and record the results internally + as the current profile. - percall - is the quotient of ``cumtime`` divided by primitive calls + .. method:: print_stats(sort=-1) - filename:lineno(function) - provides the respective data of each function + Create a :class:`~pstats.Stats` object based on the current + profile and print the results to stdout. - When there are two numbers in the first column (for example, - ``43/3``), then the latter is the number of primitive calls, and - the former is the actual number of calls. Note that when the - function does not recurse, these two values are the same, and only - the single figure is printed. For information on the sort - argument, refer to :meth:`pstats.Stats.sort_stats`. + .. method:: dump_stats(filename) + Write the results of the current profile to *filename*. -.. function:: runctx(command, globals, locals[, filename]) + .. method:: run(cmd) - This function is similar to :func:`run`, with added arguments to supply the - globals and locals dictionaries for the *command* string. + Profile the cmd via :func:`exec`. -Analysis of the profiler data is done using the :class:`Stats` class. + .. method:: runctx(cmd, globals, locals) -.. note:: + Profile the cmd via :func:`exec` with the specified global and + local environment. + + .. method:: runcall(func, *args, **kwargs) + + Profile ``func(*args, **kwargs)`` + +.. _profile-stats: - The :class:`Stats` class is defined in the :mod:`pstats` module. +The :class:`Stats` Class +======================== +Analysis of the profiler data is done using the :class:`~pstats.Stats` class. .. module:: pstats :synopsis: Statistics object for use with the profiler. +.. class:: Stats(*filenames or profile, stream=sys.stdout) -.. class:: Stats(filename, stream=sys.stdout[, ...]) + This class constructor creates an instance of a "statistics object" from a + *filename* (or list of filenames) or from a :class:`Profile` instance. Output + will be printed to the stream specified by *stream*. - This class constructor creates an instance of a "statistics object" - from a *filename* (or set of filenames). :class:`Stats` objects - are manipulated by methods, in order to print useful reports. You - may specify an alternate output stream by giving the keyword - argument, ``stream``. + The file selected by the above constructor must have been created by the + corresponding version of :mod:`profile` or :mod:`cProfile`. To be specific, + there is *no* file compatibility guaranteed with future versions of this + profiler, and there is no compatibility with files produced by other + profilers. If several files are provided, all the statistics for identical + functions will be coalesced, so that an overall view of several processes can + be considered in a single report. If additional files need to be combined + with data in an existing :class:`~pstats.Stats` object, the + :meth:`~pstats.Stats.add` method can be used. - The file selected by the above constructor must have been created - by the corresponding version of :mod:`profile` or :mod:`cProfile`. - To be specific, there is *no* file compatibility guaranteed with - future versions of this profiler, and there is no compatibility - with files produced by other profilers. If several files are - provided, all the statistics for identical functions will be - coalesced, so that an overall view of several processes can be - considered in a single report. If additional files need to be - combined with data in an existing :class:`Stats` object, the - :meth:`add` method can be used. + Instead of reading the profile data from a file, a :class:`cProfile.Profile` + or :class:`profile.Profile` object can be used as the profile data source. - .. (such as the old system profiler). + :class:`Stats` objects have the following methods: - .. versionchanged:: 2.5 - The *stream* parameter was added. + .. method:: strip_dirs() + This method for the :class:`Stats` class removes all leading path + information from file names. It is very useful in reducing the size of + the printout to fit within (close to) 80 columns. This method modifies + the object, and the stripped information is lost. After performing a + strip operation, the object is considered to have its entries in a + "random" order, as it was just after object initialization and loading. + If :meth:`~pstats.Stats.strip_dirs` causes two function names to be + indistinguishable (they are on the same line of the same filename, and + have the same function name), then the statistics for these two entries + are accumulated into a single entry. -.. _profile-stats: -The :class:`Stats` Class ------------------------- + .. method:: add(*filenames) -:class:`Stats` objects have the following methods: + This method of the :class:`Stats` class accumulates additional profiling + information into the current profiling object. Its arguments should refer + to filenames created by the corresponding version of :func:`profile.run` + or :func:`cProfile.run`. Statistics for identically named (re: file, line, + name) functions are automatically accumulated into single function + statistics. -.. method:: Stats.strip_dirs() + .. method:: dump_stats(filename) - This method for the :class:`Stats` class removes all leading path - information from file names. It is very useful in reducing the - size of the printout to fit within (close to) 80 columns. This - method modifies the object, and the stripped information is lost. - After performing a strip operation, the object is considered to - have its entries in a "random" order, as it was just after object - initialization and loading. If :meth:`strip_dirs` causes two - function names to be indistinguishable (they are on the same line - of the same filename, and have the same function name), then the - statistics for these two entries are accumulated into a single - entry. + Save the data loaded into the :class:`Stats` object to a file named + *filename*. The file is created if it does not exist, and is overwritten + if it already exists. This is equivalent to the method of the same name + on the :class:`profile.Profile` and :class:`cProfile.Profile` classes. + .. versionadded:: 2.3 -.. method:: Stats.add(filename[, ...]) - This method of the :class:`Stats` class accumulates additional profiling - information into the current profiling object. Its arguments should refer to - filenames created by the corresponding version of :func:`profile.run` or - :func:`cProfile.run`. Statistics for identically named (re: file, line, name) - functions are automatically accumulated into single function statistics. + .. method:: sort_stats(*keys) + + This method modifies the :class:`Stats` object by sorting it according to + the supplied criteria. The argument is typically a string identifying the + basis of a sort (example: ``'time'`` or ``'name'``). + + When more than one key is provided, then additional keys are used as + secondary criteria when there is equality in all keys selected before + them. For example, ``sort_stats('name', 'file')`` will sort all the + entries according to their function name, and resolve all ties (identical + function names) by sorting by file name. + + Abbreviations can be used for any key names, as long as the abbreviation + is unambiguous. The following are the keys currently defined: + + +------------------+----------------------+ + | Valid Arg | Meaning | + +==================+======================+ + | ``'calls'`` | call count | + +------------------+----------------------+ + | ``'cumulative'`` | cumulative time | + +------------------+----------------------+ + | ``'cumtime'`` | cumulative time | + +------------------+----------------------+ + | ``'file'`` | file name | + +------------------+----------------------+ + | ``'filename'`` | file name | + +------------------+----------------------+ + | ``'module'`` | file name | + +------------------+----------------------+ + | ``'ncalls'`` | call count | + +------------------+----------------------+ + | ``'pcalls'`` | primitive call count | + +------------------+----------------------+ + | ``'line'`` | line number | + +------------------+----------------------+ + | ``'name'`` | function name | + +------------------+----------------------+ + | ``'nfl'`` | name/file/line | + +------------------+----------------------+ + | ``'stdname'`` | standard name | + +------------------+----------------------+ + | ``'time'`` | internal time | + +------------------+----------------------+ + | ``'tottime'`` | internal time | + +------------------+----------------------+ + + Note that all sorts on statistics are in descending order (placing most + time consuming items first), where as name, file, and line number searches + are in ascending order (alphabetical). The subtle distinction between + ``'nfl'`` and ``'stdname'`` is that the standard name is a sort of the + name as printed, which means that the embedded line numbers get compared + in an odd way. For example, lines 3, 20, and 40 would (if the file names + were the same) appear in the string order 20, 3 and 40. In contrast, + ``'nfl'`` does a numeric compare of the line numbers. In fact, + ``sort_stats('nfl')`` is the same as ``sort_stats('name', 'file', + 'line')``. + + For backward-compatibility reasons, the numeric arguments ``-1``, ``0``, + ``1``, and ``2`` are permitted. They are interpreted as ``'stdname'``, + ``'calls'``, ``'time'``, and ``'cumulative'`` respectively. If this old + style format (numeric) is used, only one sort key (the numeric key) will + be used, and additional arguments will be silently ignored. + + .. For compatibility with the old profiler. + + + .. method:: reverse_order() + + This method for the :class:`Stats` class reverses the ordering of the + basic list within the object. Note that by default ascending vs + descending order is properly selected based on the sort key of choice. + + .. This method is provided primarily for compatibility with the old + profiler. + + + .. method:: print_stats(*restrictions) + + This method for the :class:`Stats` class prints out a report as described + in the :func:`profile.run` definition. + + The order of the printing is based on the last + :meth:`~pstats.Stats.sort_stats` operation done on the object (subject to + caveats in :meth:`~pstats.Stats.add` and + :meth:`~pstats.Stats.strip_dirs`). + + The arguments provided (if any) can be used to limit the list down to the + significant entries. Initially, the list is taken to be the complete set + of profiled functions. Each restriction is either an integer (to select a + count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to + select a percentage of lines), or a regular expression (to pattern match + the standard name that is printed. If several restrictions are provided, + then they are applied sequentially. For example:: + + print_stats(.1, 'foo:') + + would first limit the printing to first 10% of list, and then only print + functions that were part of filename :file:`.\*foo:`. In contrast, the + command:: + + print_stats('foo:', .1) + + would limit the list to all functions having file names :file:`.\*foo:`, + and then proceed to only print the first 10% of them. + + + .. method:: print_callers(*restrictions) + + This method for the :class:`Stats` class prints a list of all functions + that called each function in the profiled database. The ordering is + identical to that provided by :meth:`~pstats.Stats.print_stats`, and the + definition of the restricting argument is also identical. Each caller is + reported on its own line. The format differs slightly depending on the + profiler that produced the stats: + + * With :mod:`profile`, a number is shown in parentheses after each caller + to show how many times this specific call was made. For convenience, a + second non-parenthesized number repeats the cumulative time spent in the + function at the right. + + * With :mod:`cProfile`, each caller is preceded by three numbers: the + number of times this specific call was made, and the total and + cumulative times spent in the current function while it was invoked by + this specific caller. + + + .. method:: print_callees(*restrictions) + This method for the :class:`Stats` class prints a list of all function + that were called by the indicated function. Aside from this reversal of + direction of calls (re: called vs was called by), the arguments and + ordering are identical to the :meth:`~pstats.Stats.print_callers` method. -.. method:: Stats.dump_stats(filename) - Save the data loaded into the :class:`Stats` object to a file named - *filename*. The file is created if it does not exist, and is - overwritten if it already exists. This is equivalent to the method - of the same name on the :class:`profile.Profile` and - :class:`cProfile.Profile` classes. +.. _deterministic-profiling: - .. versionadded:: 2.3 +What Is Deterministic Profiling? +================================ +:dfn:`Deterministic profiling` is meant to reflect the fact that all *function +call*, *function return*, and *exception* events are monitored, and precise +timings are made for the intervals between these events (during which time the +user's code is executing). In contrast, :dfn:`statistical profiling` (which is +not done by this module) randomly samples the effective instruction pointer, and +deduces where time is being spent. The latter technique traditionally involves +less overhead (as the code does not need to be instrumented), but provides only +relative indications of where time is being spent. -.. method:: Stats.sort_stats(key[, ...]) - - This method modifies the :class:`Stats` object by sorting it - according to the supplied criteria. The argument is typically a - string identifying the basis of a sort (example: ``'time'`` or - ``'name'``). - - When more than one key is provided, then additional keys are used - as secondary criteria when there is equality in all keys selected - before them. For example, ``sort_stats('name', 'file')`` will sort - all the entries according to their function name, and resolve all - ties (identical function names) by sorting by file name. - - Abbreviations can be used for any key names, as long as the abbreviation is - unambiguous. The following are the keys currently defined: - - +------------------+----------------------+ - | Valid Arg | Meaning | - +==================+======================+ - | ``'calls'`` | call count | - +------------------+----------------------+ - | ``'cumulative'`` | cumulative time | - +------------------+----------------------+ - | ``'cumtime'`` | cumulative time | - +------------------+----------------------+ - | ``'file'`` | file name | - +------------------+----------------------+ - | ``'filename'`` | file name | - +------------------+----------------------+ - | ``'module'`` | file name | - +------------------+----------------------+ - | ``'ncalls'`` | call count | - +------------------+----------------------+ - | ``'pcalls'`` | primitive call count | - +------------------+----------------------+ - | ``'line'`` | line number | - +------------------+----------------------+ - | ``'name'`` | function name | - +------------------+----------------------+ - | ``'nfl'`` | name/file/line | - +------------------+----------------------+ - | ``'stdname'`` | standard name | - +------------------+----------------------+ - | ``'time'`` | internal time | - +------------------+----------------------+ - | ``'tottime'`` | internal time | - +------------------+----------------------+ - - Note that all sorts on statistics are in descending order (placing - most time consuming items first), where as name, file, and line - number searches are in ascending order (alphabetical). The subtle - distinction between ``'nfl'`` and ``'stdname'`` is that the - standard name is a sort of the name as printed, which means that - the embedded line numbers get compared in an odd way. For example, - lines 3, 20, and 40 would (if the file names were the same) appear - in the string order 20, 3 and 40. In contrast, ``'nfl'`` does a - numeric compare of the line numbers. In fact, - ``sort_stats('nfl')`` is the same as ``sort_stats('name', 'file', - 'line')``. - - For backward-compatibility reasons, the numeric arguments ``-1``, - ``0``, ``1``, and ``2`` are permitted. They are interpreted as - ``'stdname'``, ``'calls'``, ``'time'``, and ``'cumulative'`` - respectively. If this old style format (numeric) is used, only one - sort key (the numeric key) will be used, and additional arguments - will be silently ignored. - - .. For compatibility with the old profiler, - - -.. method:: Stats.reverse_order() - - This method for the :class:`Stats` class reverses the ordering of - the basic list within the object. Note that by default ascending - vs descending order is properly selected based on the sort key of - choice. - - .. This method is provided primarily for compatibility with the old profiler. - - -.. method:: Stats.print_stats([restriction, ...]) - - This method for the :class:`Stats` class prints out a report as - described in the :func:`profile.run` definition. - - The order of the printing is based on the last :meth:`sort_stats` - operation done on the object (subject to caveats in :meth:`add` and - :meth:`strip_dirs`). - - The arguments provided (if any) can be used to limit the list down - to the significant entries. Initially, the list is taken to be the - complete set of profiled functions. Each restriction is either an - integer (to select a count of lines), or a decimal fraction between - 0.0 and 1.0 inclusive (to select a percentage of lines), or a - regular expression (to pattern match the standard name that is - printed; as of Python 1.5b1, this uses the Perl-style regular - expression syntax defined by the :mod:`re` module). If several - restrictions are provided, then they are applied sequentially. For - example:: - - print_stats(.1, 'foo:') - - would first limit the printing to first 10% of list, and then only print - functions that were part of filename :file:`.\*foo:`. In contrast, the - command:: - - print_stats('foo:', .1) - - would limit the list to all functions having file names :file:`.\*foo:`, and - then proceed to only print the first 10% of them. - - -.. method:: Stats.print_callers([restriction, ...]) - - This method for the :class:`Stats` class prints a list of all functions that - called each function in the profiled database. The ordering is identical to - that provided by :meth:`print_stats`, and the definition of the restricting - argument is also identical. Each caller is reported on its own line. The - format differs slightly depending on the profiler that produced the stats: - - * With :mod:`profile`, a number is shown in parentheses after each caller to - show how many times this specific call was made. For convenience, a second - non-parenthesized number repeats the cumulative time spent in the function - at the right. - - * With :mod:`cProfile`, each caller is preceded by three numbers: - the number of times this specific call was made, and the total - and cumulative times spent in the current function while it was - invoked by this specific caller. - - -.. method:: Stats.print_callees([restriction, ...]) +In Python, since there is an interpreter active during execution, the presence +of instrumented code is not required to do deterministic profiling. Python +automatically provides a :dfn:`hook` (optional callback) for each event. In +addition, the interpreted nature of Python tends to add so much overhead to +execution, that deterministic profiling tends to only add small processing +overhead in typical applications. The result is that deterministic profiling is +not that expensive, yet provides extensive run time statistics about the +execution of a Python program. - This method for the :class:`Stats` class prints a list of all - function that were called by the indicated function. Aside from - this reversal of direction of calls (re: called vs was called by), - the arguments and ordering are identical to the - :meth:`print_callers` method. +Call count statistics can be used to identify bugs in code (surprising counts), +and to identify possible inline-expansion points (high call counts). Internal +time statistics can be used to identify "hot loops" that should be carefully +optimized. Cumulative time statistics should be used to identify high level +errors in the selection of algorithms. Note that the unusual handling of +cumulative times in this profiler allows statistics for recursive +implementations of algorithms to be directly compared to iterative +implementations. -.. _profile-limits: +.. _profile-limitations: Limitations =========== @@ -554,7 +587,7 @@ The profiler of the :mod:`profile` module subtracts a constant from each event handling time to compensate for the overhead of calling the time function, and socking away the results. By default, the constant is 0. The following procedure can be used to obtain a better constant for a given platform (see -discussion in section Limitations above). :: +:ref:`profile-limitations`). :: import profile pr = profile.Profile() @@ -564,8 +597,8 @@ discussion in section Limitations above). :: The method executes the number of Python calls given by the argument, directly and again under the profiler, measuring the time for both. It then computes the hidden overhead per profiler event, and returns that as a float. For example, -on an 800 MHz Pentium running Windows 2000, and using Python's time.clock() as -the timer, the magical number is about 12.5e-6. +on a 1.8Ghz Intel Core i5 running Mac OS X, and using Python's time.clock() as +the timer, the magical number is about 4.04e-6. The object of this exercise is to get a fairly consistent result. If your computer is *very* fast, or your timer function has poor resolution, you might @@ -588,57 +621,50 @@ When you have a consistent answer, there are three ways you can use it: [#]_ :: If you have a choice, you are better off choosing a smaller constant, and then your results will "less often" show up as negative in profile statistics. +.. _profile-timers: -.. _profiler-extensions: - -Extensions --- Deriving Better Profilers -======================================== - -The :class:`Profile` class of both modules, :mod:`profile` and :mod:`cProfile`, -were written so that derived classes could be developed to extend the profiler. -The details are not described here, as doing this successfully requires an -expert understanding of how the :class:`Profile` class works internally. Study -the source code of the module carefully if you want to pursue this. +Using a customer timer +====================== -If all you want to do is change how current time is determined (for example, to -force use of wall-clock time or elapsed process time), pass the timing function -you want to the :class:`Profile` class constructor:: +If you want to change how current time is determined (for example, to force use +of wall-clock time or elapsed process time), pass the timing function you want +to the :class:`Profile` class constructor:: - pr = profile.Profile(your_time_func) + pr = profile.Profile(your_time_func) -The resulting profiler will then call :func:`your_time_func`. +The resulting profiler will then call ``your_time_func``. Depending on whether +you are using :class:`profile.Profile` or :class:`cProfile.Profile`, +``your_time_func``'s return value will be interpreted differently: :class:`profile.Profile` - :func:`your_time_func` should return a single number, or a list of - numbers whose sum is the current time (like what :func:`os.times` - returns). If the function returns a single time number, or the - list of returned numbers has length 2, then you will get an - especially fast version of the dispatch routine. - - Be warned that you should calibrate the profiler class for the - timer function that you choose. For most machines, a timer that - returns a lone integer value will provide the best results in terms - of low overhead during profiling. (:func:`os.times` is *pretty* - bad, as it returns a tuple of floating point values). If you want - to substitute a better timer in the cleanest fashion, derive a - class and hardwire a replacement dispatch method that best handles - your timer call, along with the appropriate calibration constant. + ``your_time_func`` should return a single number, or a list of numbers whose + sum is the current time (like what :func:`os.times` returns). If the + function returns a single time number, or the list of returned numbers has + length 2, then you will get an especially fast version of the dispatch + routine. + + Be warned that you should calibrate the profiler class for the timer function + that you choose (see :ref:`profile-calibration`). For most machines, a timer + that returns a lone integer value will provide the best results in terms of + low overhead during profiling. (:func:`os.times` is *pretty* bad, as it + returns a tuple of floating point values). If you want to substitute a + better timer in the cleanest fashion, derive a class and hardwire a + replacement dispatch method that best handles your timer call, along with the + appropriate calibration constant. :class:`cProfile.Profile` - :func:`your_time_func` should return a single number. If it - returns plain integers, you can also invoke the class constructor - with a second argument specifying the real duration of one unit of - time. For example, if :func:`your_integer_time_func` returns times - measured in thousands of seconds, you would construct the - :class:`Profile` instance as follows:: - - pr = profile.Profile(your_integer_time_func, 0.001) - - As the :mod:`cProfile.Profile` class cannot be calibrated, custom - timer functions should be used with care and should be as fast as - possible. For the best results with a custom timer, it might be - necessary to hard-code it in the C source of the internal - :mod:`_lsprof` module. + ``your_time_func`` should return a single number. If it returns integers, + you can also invoke the class constructor with a second argument specifying + the real duration of one unit of time. For example, if + ``your_integer_time_func`` returns times measured in thousands of seconds, + you would construct the :class:`Profile` instance as follows:: + + pr = cProfile.Profile(your_integer_time_func, 0.001) + + As the :mod:`cProfile.Profile` class cannot be calibrated, custom timer + functions should be used with care and should be as fast as possible. For + the best results with a custom timer, it might be necessary to hard-code it + in the C source of the internal :mod:`_lsprof` module. .. rubric:: Footnotes diff --git a/Misc/ACKS b/Misc/ACKS index 12507e41203..4de44cc9ae6 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -791,6 +791,7 @@ Jim St. Pierre Dan Pierson Martijn Pieters François Pinard +Tom Pinckney Zach Pincus Michael Piotrowski Antoine Pitrou diff --git a/Misc/NEWS b/Misc/NEWS index a04210c21b5..62e809a48c5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -65,6 +65,9 @@ Documentation - Issue #15940: Specify effect of locale on time functions. +- Issue #6696: add documentation for the Profile objects, and improve + profile/cProfile docs. Patch by Tom Pinckney. + What's New in Python 2.7.4? =========================== From 4ac426665c19bc5a7004cee6f57bfa63c100036b Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 12 Apr 2013 23:39:33 +0300 Subject: [PATCH 035/480] Issue #13355: Raise ValueError on random.triangular call with invalid params. Initial patch by Yuriy Senko. --- Lib/random.py | 10 ++++++++++ Lib/test/test_random.py | 29 ++++++++++++++++++++++++++++- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/Lib/random.py b/Lib/random.py index af04ab207d3..8560fb9a25d 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -367,6 +367,16 @@ def triangular(self, low=0.0, high=1.0, mode=None): http://en.wikipedia.org/wiki/Triangular_distribution """ + # Sanity check. According to the doc low must be less or equal to + # high. And mode should be somewhere between these bounds. + if low > high: + raise ValueError('high cannot be less then low.') + if mode is not None and (mode < low or mode > high): + raise ValueError('mode must be between low and high.') + + if high == low: + return low + u = self.random() c = 0.5 if mode is None else (mode - low) / (high - low) if u > c: diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 33164150c36..75e1afc8247 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -43,6 +43,33 @@ def test_seedargs(self): self.assertRaises(TypeError, self.gen.seed, 1, 2) self.assertRaises(TypeError, type(self.gen), []) + def test_triangular(self): + # Check that triangular() correctly handles bad input. See issue 13355. + # mode > high. + with self.assertRaises(ValueError): + random.triangular(mode=2) + with self.assertRaises(ValueError): + random.triangular(low=1, high=10, mode=11) + with self.assertRaises(ValueError): + random.triangular(low=1, high=1, mode=11) + # mode < low. + with self.assertRaises(ValueError): + random.triangular(mode=-1) + with self.assertRaises(ValueError): + random.triangular(low=1, high=10, mode=0) + with self.assertRaises(ValueError): + random.triangular(low=1, high=1, mode=0) + # low > high + with self.assertRaises(ValueError): + random.triangular(low=5, high=2) + with self.assertRaises(ValueError): + random.triangular(low=5, high=2, mode=1) + with self.assertRaises(ValueError): + random.triangular(low=-2, high=-5) + + self.assertEqual(random.triangular(low=10, high=10), 10) + self.assertEqual(random.triangular(low=10, high=10, mode=10), 10) + def test_jumpahead(self): self.gen.seed() state1 = self.gen.getstate() @@ -543,7 +570,7 @@ def test_constant(self): for variate, args, expected in [ (g.uniform, (10.0, 10.0), 10.0), (g.triangular, (10.0, 10.0), 10.0), - #(g.triangular, (10.0, 10.0, 10.0), 10.0), + (g.triangular, (10.0, 10.0, 10.0), 10.0), (g.expovariate, (float('inf'),), 0.0), (g.vonmisesvariate, (3.0, float('inf')), 3.0), (g.gauss, (10.0, 0.0), 10.0), diff --git a/Misc/ACKS b/Misc/ACKS index 4de44cc9ae6..9480f2d3d33 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -907,6 +907,7 @@ Nick Seidenman Žiga Seilnach Yury Selivanov Fred Sells +Yuriy Senko Jiwon Seo Joakim Sernbrant Roger Serwy diff --git a/Misc/NEWS b/Misc/NEWS index 62e809a48c5..c4d9635a8b3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -22,6 +22,9 @@ Core and Builtins Library ------- +- Issue #13355: Raise ValueError on random.triangular call with invalid params. + Initial patch by Yuriy Senko. + - Issue #17666: Fix reading gzip files with an extra field. - Issue #13150, #17512: sysconfig no longer parses the Makefile and config.h From 108a7a316844420f728c72d0aabf6ef440afb964 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 13 Apr 2013 12:28:17 +0300 Subject: [PATCH 036/480] Issue #17656: Fix extraction of zip files with unicode member paths. --- Lib/test/test_zipfile.py | 21 ++++++++++++++++++++- Lib/zipfile.py | 5 ++++- Misc/NEWS | 2 ++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index c26ca80b0a9..f960b6c7d4f 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -18,7 +18,7 @@ from random import randint, random from unittest import skipUnless -from test.test_support import TESTFN, run_unittest, findfile, unlink +from test.test_support import TESTFN, TESTFN_UNICODE, run_unittest, findfile, unlink TESTFN2 = TESTFN + "2" TESTFNDIR = TESTFN + "d" @@ -424,6 +424,25 @@ def check_file(self, filename, content): with open(filename, 'rb') as f: self.assertEqual(f.read(), content) + def test_extract_unicode_filenames(self): + fnames = [u'foo.txt', os.path.basename(TESTFN_UNICODE)] + content = 'Test for unicode filename' + with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: + for fname in fnames: + zipfp.writestr(fname, content) + + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + for fname in fnames: + writtenfile = zipfp.extract(fname) + + # make sure it was written to the right place + correctfile = os.path.join(os.getcwd(), fname) + correctfile = os.path.normpath(correctfile) + self.assertEqual(writtenfile, correctfile) + + self.check_file(writtenfile, content) + os.remove(writtenfile) + def test_extract_hackers_arcnames(self): hacknames = [ ('../foo/bar', 'foo/bar'), diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 9d1a98479db..6639317537b 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -1053,7 +1053,10 @@ def _extract_member(self, member, targetpath, pwd): if os.path.sep == '\\': # filter illegal characters on Windows illegal = ':<>|"?*' - table = string.maketrans(illegal, '_' * len(illegal)) + if isinstance(arcname, unicode): + table = {ord(c): ord('_') for c in illegal} + else: + table = string.maketrans(illegal, '_' * len(illegal)) arcname = arcname.translate(table) # remove trailing dots arcname = (x.rstrip('.') for x in arcname.split(os.path.sep)) diff --git a/Misc/NEWS b/Misc/NEWS index c4d9635a8b3..ee5393c2867 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -22,6 +22,8 @@ Core and Builtins Library ------- +- Issue #17656: Fix extraction of zip files with unicode member paths. + - Issue #13355: Raise ValueError on random.triangular call with invalid params. Initial patch by Yuriy Senko. From d00cbd02d744d415fbb7bad3139e191fcfd4aa3d Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sat, 13 Apr 2013 15:19:05 +0100 Subject: [PATCH 037/480] Issue #16447: Fix potential segfault when setting __name__ on a class. --- Lib/test/test_descr.py | 14 ++++++++++++++ Misc/NEWS | 3 +++ Objects/typeobject.c | 6 +++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index bfb3552f55a..07664ac208d 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4136,6 +4136,20 @@ class C(object): C.__name__ = 'D.E' self.assertEqual((C.__module__, C.__name__), (mod, 'D.E')) + def test_evil_type_name(self): + # A badly placed Py_DECREF in type_set_name led to arbitrary code + # execution while the type structure was not in a sane state, and a + # possible segmentation fault as a result. See bug #16447. + class Nasty(str): + def __del__(self): + C.__name__ = "other" + + class C(object): + pass + + C.__name__ = Nasty("abc") + C.__name__ = "normal" + def test_subclass_right_op(self): # Testing correct dispatch of subclass overloading __r__... diff --git a/Misc/NEWS b/Misc/NEWS index ee5393c2867..cb0dfc81e73 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,9 @@ Build Core and Builtins ----------------- +- Issue #16447: Fixed potential segmentation fault when setting __name__ on a + class. + - Issue #17610: Don't rely on non-standard behavior of the C qsort() function. Library diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 6c88f53af42..be04c9e9140 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -225,6 +225,7 @@ static int type_set_name(PyTypeObject *type, PyObject *value, void *context) { PyHeapTypeObject* et; + PyObject *tmp; if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { PyErr_Format(PyExc_TypeError, @@ -253,10 +254,13 @@ type_set_name(PyTypeObject *type, PyObject *value, void *context) Py_INCREF(value); - Py_DECREF(et->ht_name); + /* Wait until et is a sane state before Py_DECREF'ing the old et->ht_name + value. (Bug #16447.) */ + tmp = et->ht_name; et->ht_name = value; type->tp_name = PyString_AS_STRING(value); + Py_DECREF(tmp); return 0; } From d7b787a10f545bcf55a787640bf6392cdb493a14 Mon Sep 17 00:00:00 2001 From: doko Date: Sat, 13 Apr 2013 17:05:11 +0200 Subject: [PATCH 038/480] - Issue #17536: Add to webbrowser's browser list: xdg-open, gvfs-open, www-browser, x-www-browser, chromium browsers, iceweasel, iceape. --- Lib/webbrowser.py | 31 +++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ 2 files changed, 34 insertions(+) diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index f3c53d423cf..d3272a0a48e 100644 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -304,6 +304,18 @@ class Galeon(UnixBrowser): background = True +class Chrome(UnixBrowser): + "Launcher class for Google Chrome browser." + + remote_args = ['%action', '%s'] + remote_action = "" + remote_action_newwin = "--new-window" + remote_action_newtab = "" + background = True + +Chromium = Chrome + + class Opera(UnixBrowser): "Launcher class for Opera browser." @@ -441,6 +453,14 @@ def open(self, url, new=0, autoraise=True): def register_X_browsers(): + # use xdg-open if around + if _iscommand("xdg-open"): + register("xdg-open", None, BackgroundBrowser("xdg-open")) + + # The default GNOME3 browser + if "GNOME_DESKTOP_SESSION_ID" in os.environ and _iscommand("gvfs-open"): + register("gvfs-open", None, BackgroundBrowser("gvfs-open")) + # The default GNOME browser if "GNOME_DESKTOP_SESSION_ID" in os.environ and _iscommand("gnome-open"): register("gnome-open", None, BackgroundBrowser("gnome-open")) @@ -449,9 +469,13 @@ def register_X_browsers(): if "KDE_FULL_SESSION" in os.environ and _iscommand("kfmclient"): register("kfmclient", Konqueror, Konqueror("kfmclient")) + if _iscommand("x-www-browser"): + register("x-www-browser", None, BackgroundBrowser("x-www-browser")) + # The Mozilla/Netscape browsers for browser in ("mozilla-firefox", "firefox", "mozilla-firebird", "firebird", + "iceweasel", "iceape", "seamonkey", "mozilla", "netscape"): if _iscommand(browser): register(browser, None, Mozilla(browser)) @@ -471,6 +495,11 @@ def register_X_browsers(): if _iscommand("skipstone"): register("skipstone", None, BackgroundBrowser("skipstone")) + # Google Chrome/Chromium browsers + for browser in ("google-chrome", "chrome", "chromium", "chromium-browser"): + if _iscommand(browser): + register(browser, None, Chrome(browser)) + # Opera, quite popular if _iscommand("opera"): register("opera", None, Opera("opera")) @@ -489,6 +518,8 @@ def register_X_browsers(): # Also try console browsers if os.environ.get("TERM"): + if _iscommand("www-browser"): + register("www-browser", None, GenericBrowser("www-browser")) # The Links/elinks browsers if _iscommand("links"): register("links", None, GenericBrowser("links")) diff --git a/Misc/NEWS b/Misc/NEWS index cb0dfc81e73..3a2c1ecf289 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -25,6 +25,9 @@ Core and Builtins Library ------- +- Issue #17536: Add to webbrowser's browser list: xdg-open, gvfs-open, + www-browser, x-www-browser, chromium browsers, iceweasel, iceape. + - Issue #17656: Fix extraction of zip files with unicode member paths. - Issue #13355: Raise ValueError on random.triangular call with invalid params. From cbd7fdfe924bad044d68a618bf2f63c94426ee5b Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 13 Apr 2013 18:05:44 +0300 Subject: [PATCH 039/480] Revert changes for #13355 by request from Raymond Hettinger --- Lib/random.py | 10 ---------- Lib/test/test_random.py | 29 +---------------------------- Misc/ACKS | 1 - Misc/NEWS | 3 --- 4 files changed, 1 insertion(+), 42 deletions(-) diff --git a/Lib/random.py b/Lib/random.py index 8560fb9a25d..af04ab207d3 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -367,16 +367,6 @@ def triangular(self, low=0.0, high=1.0, mode=None): http://en.wikipedia.org/wiki/Triangular_distribution """ - # Sanity check. According to the doc low must be less or equal to - # high. And mode should be somewhere between these bounds. - if low > high: - raise ValueError('high cannot be less then low.') - if mode is not None and (mode < low or mode > high): - raise ValueError('mode must be between low and high.') - - if high == low: - return low - u = self.random() c = 0.5 if mode is None else (mode - low) / (high - low) if u > c: diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 75e1afc8247..33164150c36 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -43,33 +43,6 @@ def test_seedargs(self): self.assertRaises(TypeError, self.gen.seed, 1, 2) self.assertRaises(TypeError, type(self.gen), []) - def test_triangular(self): - # Check that triangular() correctly handles bad input. See issue 13355. - # mode > high. - with self.assertRaises(ValueError): - random.triangular(mode=2) - with self.assertRaises(ValueError): - random.triangular(low=1, high=10, mode=11) - with self.assertRaises(ValueError): - random.triangular(low=1, high=1, mode=11) - # mode < low. - with self.assertRaises(ValueError): - random.triangular(mode=-1) - with self.assertRaises(ValueError): - random.triangular(low=1, high=10, mode=0) - with self.assertRaises(ValueError): - random.triangular(low=1, high=1, mode=0) - # low > high - with self.assertRaises(ValueError): - random.triangular(low=5, high=2) - with self.assertRaises(ValueError): - random.triangular(low=5, high=2, mode=1) - with self.assertRaises(ValueError): - random.triangular(low=-2, high=-5) - - self.assertEqual(random.triangular(low=10, high=10), 10) - self.assertEqual(random.triangular(low=10, high=10, mode=10), 10) - def test_jumpahead(self): self.gen.seed() state1 = self.gen.getstate() @@ -570,7 +543,7 @@ def test_constant(self): for variate, args, expected in [ (g.uniform, (10.0, 10.0), 10.0), (g.triangular, (10.0, 10.0), 10.0), - (g.triangular, (10.0, 10.0, 10.0), 10.0), + #(g.triangular, (10.0, 10.0, 10.0), 10.0), (g.expovariate, (float('inf'),), 0.0), (g.vonmisesvariate, (3.0, float('inf')), 3.0), (g.gauss, (10.0, 0.0), 10.0), diff --git a/Misc/ACKS b/Misc/ACKS index 9480f2d3d33..4de44cc9ae6 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -907,7 +907,6 @@ Nick Seidenman Žiga Seilnach Yury Selivanov Fred Sells -Yuriy Senko Jiwon Seo Joakim Sernbrant Roger Serwy diff --git a/Misc/NEWS b/Misc/NEWS index cb0dfc81e73..a5b78db933d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,9 +27,6 @@ Library - Issue #17656: Fix extraction of zip files with unicode member paths. -- Issue #13355: Raise ValueError on random.triangular call with invalid params. - Initial patch by Yuriy Senko. - - Issue #17666: Fix reading gzip files with an extra field. - Issue #13150, #17512: sysconfig no longer parses the Makefile and config.h From 209cfa0e2402743787f1d667be50bd510e124174 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 13 Apr 2013 20:07:42 +0300 Subject: [PATCH 040/480] #17686: fix broken link in Doc/using/unix.rst. Patch by Dan Riti. --- Doc/using/unix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index cb690a6b630..ec5ac6c65d5 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -28,7 +28,7 @@ following links: http://www.debian.org/doc/manuals/maint-guide/first.en.html for Debian users - http://linuxmafia.com/pub/linux/suse-linux-internals/chapter35.html + http://en.opensuse.org/Portal:Packaging for OpenSuse users http://docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch-creating-rpms.html for Fedora users From 0b5766f04e9df54e4354b561dd1e266133258001 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 13 Apr 2013 21:15:10 +0300 Subject: [PATCH 041/480] Issue #17016: Get rid of possible pointer wraparounds and integer overflows in the re module. Patch by Nickolai Zeldovich. --- Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/_sre.c | 22 +++++++++++----------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS index 4de44cc9ae6..5779332ce35 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1130,6 +1130,7 @@ Milan Zamazal Artur Zaprzala Mike Zarnstorff Siebren van der Zee +Nickolai Zeldovich Uwe Zessin Cheng Zhang Tarek Ziadé diff --git a/Misc/NEWS b/Misc/NEWS index aea5b1342d2..1dd72fbbd83 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -25,6 +25,9 @@ Core and Builtins Library ------- +- Issue #17016: Get rid of possible pointer wraparounds and integer overflows + in the re module. Patch by Nickolai Zeldovich. + - Issue #17536: Add to webbrowser's browser list: xdg-open, gvfs-open, www-browser, x-www-browser, chromium browsers, iceweasel, iceape. diff --git a/Modules/_sre.c b/Modules/_sre.c index f3a29931a7a..478416fb1da 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -686,7 +686,7 @@ do { \ alloc_pos = state->data_stack_base; \ TRACE(("allocating %s in %d (%d)\n", \ SFY(type), alloc_pos, sizeof(type))); \ - if (state->data_stack_size < alloc_pos+sizeof(type)) { \ + if (sizeof(type) > state->data_stack_size - alloc_pos) { \ int j = data_stack_grow(state, sizeof(type)); \ if (j < 0) return j; \ if (ctx_pos != -1) \ @@ -706,7 +706,7 @@ do { \ do { \ TRACE(("copy data in %p to %d (%d)\n", \ data, state->data_stack_base, size)); \ - if (state->data_stack_size < state->data_stack_base+size) { \ + if (size > state->data_stack_size - state->data_stack_base) { \ int j = data_stack_grow(state, size); \ if (j < 0) return j; \ if (ctx_pos != -1) \ @@ -1028,7 +1028,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) TRACE(("|%p|%p|REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1], ctx->pattern[2])); - if (ctx->ptr + ctx->pattern[1] > end) + if (ctx->pattern[1] > end - ctx->ptr) RETURN_FAILURE; /* cannot match */ state->ptr = ctx->ptr; @@ -1111,7 +1111,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) TRACE(("|%p|%p|MIN_REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1], ctx->pattern[2])); - if (ctx->ptr + ctx->pattern[1] > end) + if (ctx->pattern[1] > end - ctx->ptr) RETURN_FAILURE; /* cannot match */ state->ptr = ctx->ptr; @@ -2784,7 +2784,7 @@ _compile(PyObject* self_, PyObject* args) skip = *code; \ VTRACE(("%lu (skip to %p)\n", \ (unsigned long)skip, code+skip)); \ - if (code+skip-adj < code || code+skip-adj > end)\ + if (skip-adj > end-code) \ FAIL; \ code++; \ } while (0) @@ -2817,7 +2817,7 @@ _validate_charset(SRE_CODE *code, SRE_CODE *end) case SRE_OP_CHARSET: offset = 32/sizeof(SRE_CODE); /* 32-byte bitmap */ - if (code+offset < code || code+offset > end) + if (offset > end-code) FAIL; code += offset; break; @@ -2825,7 +2825,7 @@ _validate_charset(SRE_CODE *code, SRE_CODE *end) case SRE_OP_BIGCHARSET: GET_ARG; /* Number of blocks */ offset = 256/sizeof(SRE_CODE); /* 256-byte table */ - if (code+offset < code || code+offset > end) + if (offset > end-code) FAIL; /* Make sure that each byte points to a valid block */ for (i = 0; i < 256; i++) { @@ -2834,7 +2834,7 @@ _validate_charset(SRE_CODE *code, SRE_CODE *end) } code += offset; offset = arg * 32/sizeof(SRE_CODE); /* 32-byte bitmap times arg */ - if (code+offset < code || code+offset > end) + if (offset > end-code) FAIL; code += offset; break; @@ -2985,11 +2985,11 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) GET_ARG; prefix_len = arg; GET_ARG; /* prefix skip */ /* Here comes the prefix string */ - if (code+prefix_len < code || code+prefix_len > newcode) + if (prefix_len > newcode-code) FAIL; code += prefix_len; /* And here comes the overlap table */ - if (code+prefix_len < code || code+prefix_len > newcode) + if (prefix_len > newcode-code) FAIL; /* Each overlap value should be < prefix_len */ for (i = 0; i < prefix_len; i++) { @@ -3118,7 +3118,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) to allow arbitrary jumps anywhere in the code; so we just look for a JUMP opcode preceding our skip target. */ - if (skip >= 3 && code+skip-3 >= code && + if (skip >= 3 && skip-3 < end-code && code[skip-3] == SRE_OP_JUMP) { VTRACE(("both then and else parts present\n")); From c388ca464c17f32f7dc5f13a40f9f6bbf9cbd8cb Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 13 Apr 2013 14:37:22 -0400 Subject: [PATCH 042/480] #2118: clarify smtplib exception documentation. --- Doc/library/smtplib.rst | 28 ++++++++++++++++------------ Lib/smtplib.py | 5 +++-- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 044bbde28b8..cad4b41389b 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -25,8 +25,9 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). A :class:`SMTP` instance encapsulates an SMTP connection. It has methods that support a full repertoire of SMTP and ESMTP operations. If the optional host and port parameters are given, the SMTP :meth:`connect` method is called - with those parameters during initialization. An :exc:`SMTPConnectError` is - raised if the specified host doesn't respond correctly. The optional + with those parameters during initialization. If the :meth:`connect` call + returns anything other than a success code, an :exc:`SMTPConnectError` is + raised. The optional *timeout* parameter specifies a timeout in seconds for blocking operations like the connection attempt (if not specified, the global default timeout setting will be used). @@ -73,7 +74,8 @@ A nice selection of exceptions is defined as well: .. exception:: SMTPException - Base exception class for all exceptions raised by this module. + The base exception class for all the other excpetions provided by this + module. .. exception:: SMTPServerDisconnected @@ -152,15 +154,6 @@ An :class:`SMTP` instance has the following methods: for connection and for all messages sent to and received from the server. -.. method:: SMTP.connect([host[, port]]) - - Connect to a host on a given port. The defaults are to connect to the local - host at the standard SMTP port (25). If the hostname ends with a colon (``':'``) - followed by a number, that suffix will be stripped off and the number - interpreted as the port number to use. This method is automatically invoked by - the constructor if a host is specified during instantiation. - - .. method:: SMTP.docmd(cmd, [, argstring]) Send a command *cmd* to the server. The optional argument *argstring* is simply @@ -177,6 +170,17 @@ An :class:`SMTP` instance has the following methods: :exc:`SMTPServerDisconnected` will be raised. +.. method:: SMTP.connect([host[, port]]) + + Connect to a host on a given port. The defaults are to connect to the local + host at the standard SMTP port (25). If the hostname ends with a colon (``':'``) + followed by a number, that suffix will be stripped off and the number + interpreted as the port number to use. This method is automatically invoked by + the constructor if a host is specified during instantiation. Returns a + 2-tuple of the response code and message sent by the server in its + connection response. + + .. method:: SMTP.helo([hostname]) Identify yourself to the SMTP server using ``HELO``. The hostname argument diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 9e8bc72379f..a3213b33d71 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -237,8 +237,9 @@ def __init__(self, host='', port=0, local_hostname=None, If specified, `host' is the name of the remote host to which to connect. If specified, `port' specifies the port to which to connect. - By default, smtplib.SMTP_PORT is used. An SMTPConnectError is raised - if the specified `host' doesn't respond correctly. If specified, + By default, smtplib.SMTP_PORT is used. If a host is specified the + connect method is called, and if it returns anything other than + a success code an SMTPConnectError is raised. If specified, `local_hostname` is used as the FQDN of the local host. By default, the local hostname is found using socket.getfqdn(). From bd94ee81ce6ef5cd031cfc68e6e6d46fc5084fab Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 13 Apr 2013 14:37:42 -0400 Subject: [PATCH 043/480] Reflow paragraph. --- Doc/library/smtplib.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index cad4b41389b..18560c129f7 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -27,10 +27,9 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). host and port parameters are given, the SMTP :meth:`connect` method is called with those parameters during initialization. If the :meth:`connect` call returns anything other than a success code, an :exc:`SMTPConnectError` is - raised. The optional - *timeout* parameter specifies a timeout in seconds for blocking operations - like the connection attempt (if not specified, the global default timeout - setting will be used). + raised. The optional *timeout* parameter specifies a timeout in seconds for + blocking operations like the connection attempt (if not specified, the + global default timeout setting will be used). For normal use, you should only require the initialization/connect, :meth:`sendmail`, and :meth:`~smtplib.quit` methods. From f67497a9c0588d36f9a1ff6b392638deceee9010 Mon Sep 17 00:00:00 2001 From: David Wolever Date: Sat, 13 Apr 2013 19:12:58 -0400 Subject: [PATCH 044/480] Improving strftime documentation. Re-ordering the table so similar directives are grouped together, adding examples, and removing some redundancy in the description of the ``%f`` formatter. --- Doc/library/datetime.rst | 246 ++++++++++++++++++++------------------- 1 file changed, 124 insertions(+), 122 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 7219cea6457..d5c62af43dc 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -551,8 +551,9 @@ Instance methods: .. method:: date.strftime(format) Return a string representing the date, controlled by an explicit format string. - Format codes referring to hours, minutes or seconds will see 0 values. See - section :ref:`strftime-strptime-behavior`. + Format codes referring to hours, minutes or seconds will see 0 values. For a + complete list of formatting directives, see section + :ref:`strftime-strptime-behavior`. .. method:: date.__format__(format) @@ -730,7 +731,8 @@ Other constructors, all class methods: *format*. This is equivalent to ``datetime(*(time.strptime(date_string, format)[0:6]))``. :exc:`ValueError` is raised if the date_string and format can't be parsed by :func:`time.strptime` or if it returns a value which isn't a - time tuple. See section :ref:`strftime-strptime-behavior`. + time tuple. For a complete list of formatting directives, see section + :ref:`strftime-strptime-behavior`. .. versionadded:: 2.5 @@ -1050,7 +1052,8 @@ Instance methods: .. method:: datetime.strftime(format) Return a string representing the date and time, controlled by an explicit format - string. See section :ref:`strftime-strptime-behavior`. + string. For a complete list of formatting directives, see section + :ref:`strftime-strptime-behavior`. .. method:: datetime.__format__(format) @@ -1283,7 +1286,8 @@ Instance methods: .. method:: time.strftime(format) Return a string representing the time, controlled by an explicit format string. - See section :ref:`strftime-strptime-behavior`. + For a complete list of formatting directives, see section + :ref:`strftime-strptime-behavior`. .. method:: time.__format__(format) @@ -1597,27 +1601,6 @@ For :class:`date` objects, the format codes for hours, minutes, seconds, and microseconds should not be used, as :class:`date` objects have no such values. If they're used anyway, ``0`` is substituted for them. -.. versionadded:: 2.6 - :class:`.time` and :class:`.datetime` objects support a ``%f`` format code - which expands to the number of microseconds in the object, zero-padded on - the left to six places. - -For a naive object, the ``%z`` and ``%Z`` format codes are replaced by empty -strings. - -For an aware object: - -``%z`` - :meth:`utcoffset` is transformed into a 5-character string of the form +HHMM or - -HHMM, where HH is a 2-digit string giving the number of UTC offset hours, and - MM is a 2-digit string giving the number of UTC offset minutes. For example, if - :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is - replaced with the string ``'-0330'``. - -``%Z`` - If :meth:`tzname` returns ``None``, ``%Z`` is replaced by an empty string. - Otherwise ``%Z`` is replaced by the returned value, which must be a string. - The full set of format codes supported varies across platforms, because Python calls the platform C library's :func:`strftime` function, and platform variations are common. @@ -1630,105 +1613,101 @@ format codes. The exact range of years for which :meth:`strftime` works also varies across platforms. Regardless of platform, years before 1900 cannot be used. -+-----------+--------------------------------+-------+ -| Directive | Meaning | Notes | -+===========+================================+=======+ -| ``%a`` | Locale's abbreviated weekday | | -| | name. | | -+-----------+--------------------------------+-------+ -| ``%A`` | Locale's full weekday name. | | -+-----------+--------------------------------+-------+ -| ``%b`` | Locale's abbreviated month | | -| | name. | | -+-----------+--------------------------------+-------+ -| ``%B`` | Locale's full month name. | | -+-----------+--------------------------------+-------+ -| ``%c`` | Locale's appropriate date and | | -| | time representation. | | -+-----------+--------------------------------+-------+ -| ``%d`` | Day of the month as a decimal | | -| | number [01,31]. | | -+-----------+--------------------------------+-------+ -| ``%f`` | Microsecond as a decimal | \(1) | -| | number [0,999999], zero-padded | | -| | on the left | | -+-----------+--------------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | | -| | decimal number [00,23]. | | -+-----------+--------------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | | -| | decimal number [01,12]. | | -+-----------+--------------------------------+-------+ -| ``%j`` | Day of the year as a decimal | | -| | number [001,366]. | | -+-----------+--------------------------------+-------+ -| ``%m`` | Month as a decimal number | | -| | [01,12]. | | -+-----------+--------------------------------+-------+ -| ``%M`` | Minute as a decimal number | | -| | [00,59]. | | -+-----------+--------------------------------+-------+ -| ``%p`` | Locale's equivalent of either | \(2) | -| | AM or PM. | | -+-----------+--------------------------------+-------+ -| ``%S`` | Second as a decimal number | \(3) | -| | [00,61]. | | -+-----------+--------------------------------+-------+ -| ``%U`` | Week number of the year | \(4) | -| | (Sunday as the first day of | | -| | the week) as a decimal number | | -| | [00,53]. All days in a new | | -| | year preceding the first | | -| | Sunday are considered to be in | | -| | week 0. | | -+-----------+--------------------------------+-------+ -| ``%w`` | Weekday as a decimal number | | -| | [0(Sunday),6]. | | -+-----------+--------------------------------+-------+ -| ``%W`` | Week number of the year | \(4) | -| | (Monday as the first day of | | -| | the week) as a decimal number | | -| | [00,53]. All days in a new | | -| | year preceding the first | | -| | Monday are considered to be in | | -| | week 0. | | -+-----------+--------------------------------+-------+ -| ``%x`` | Locale's appropriate date | | -| | representation. | | -+-----------+--------------------------------+-------+ -| ``%X`` | Locale's appropriate time | | -| | representation. | | -+-----------+--------------------------------+-------+ -| ``%y`` | Year without century as a | | -| | decimal number [00,99]. | | -+-----------+--------------------------------+-------+ -| ``%Y`` | Year with century as a decimal | | -| | number. | | -+-----------+--------------------------------+-------+ -| ``%z`` | UTC offset in the form +HHMM | \(5) | -| | or -HHMM (empty string if the | | -| | the object is naive). | | -+-----------+--------------------------------+-------+ -| ``%Z`` | Time zone name (empty string | | -| | if the object is naive). | | -+-----------+--------------------------------+-------+ -| ``%%`` | A literal ``'%'`` character. | | -+-----------+--------------------------------+-------+ ++-----------+--------------------------------+------------------------+-------+ +| Directive | Meaning | Example | Notes | ++===========+================================+========================+=======+ +| ``%a`` | Weekday as locale's | Sun, Mon, ..., Sat | | +| | abbreviated name. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%A`` | Weekday as locale's full name. | Sunday, Monday, ..., | | +| | | Saturday | | ++-----------+--------------------------------+------------------------+-------+ +| ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | +| | where 0 is Sunday and 6 is | | | +| | Saturday. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%d`` | Day of the month as a | 01, 02, ..., 31 | | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%j`` | Day of the year as a | 001, 002, ..., 366 | | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%b`` | Month as locale's abbreviated | Jan, Feb, ..., Dec | | +| | name. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%B`` | Month as locale's full name. | January, February, | | +| | | ..., December | | ++-----------+--------------------------------+------------------------+-------+ +| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | | +| | decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%y`` | Year without century as a | 00, 01, ..., 99 | | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%Y`` | Year with century as a decimal | 1970, 1988, 2001, 2013 | | +| | number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%p`` | Locale's equivalent of either | AM, PM | \(1) | +| | AM or PM. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | | +| | decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%S`` | Second as a zero-padded | 00, 01, ..., 61 | \(2) | +| | decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(3) | +| | number, zero-padded on the | 999999 | | +| | left. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%z`` | UTC offset in the form +HHMM | (empty), +0000, -0400, | \(4) | +| | or -HHMM (empty string if the | +1030 | | +| | the object is naive). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%Z`` | Time zone name (empty string | (empty), UTC, EST, CST | | +| | if the object is naive). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(5) | +| | (Sunday as the first day of | | | +| | the week) as a zero padded | | | +| | decimal number. All days in a | | | +| | new year preceding the first | | | +| | Sunday are considered to be in | | | +| | week 0. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(5) | +| | (Monday as the first day of | | | +| | the week) as a decimal number. | | | +| | All days in a new year | | | +| | preceding the first Monday | | | +| | are considered to be in | | | +| | week 0. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%c`` | Locale's appropriate date and | Mon Aug 1 16:00:00 | | +| | time representation. | 1988 | | ++-----------+--------------------------------+------------------------+-------+ +| ``%x`` | Locale's appropriate date | 08/16/88 | | +| | representation. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%X`` | Locale's appropriate time | 16:00:00 | | +| | representation. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%%`` | A literal ``'%'`` character. | % | | ++-----------+--------------------------------+------------------------+-------+ Notes: (1) - When used with the :meth:`strptime` method, the ``%f`` directive - accepts from one to six digits and zero pads on the right. ``%f`` is - an extension to the set of format characters in the C standard (but - implemented separately in datetime objects, and therefore always - available). - -(2) When used with the :meth:`strptime` method, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. -(3) +(2) The range really is ``0`` to ``61``; according to the Posix standard this accounts for leap seconds and the (very rare) double leap seconds. The :mod:`time` module may produce and does accept leap seconds since @@ -1736,13 +1715,36 @@ Notes: does not accept leap seconds in :meth:`strptime` input nor will it produce them in :func:`strftime` output. +(3) + ``%f`` is an extension to the set of format characters in the C standard + (but implemented separately in datetime objects, and therefore always + available). When used with the :meth:`strptime` method, the ``%f`` + directive accepts from one to six digits and zero pads on the right. + + .. versionadded:: 2.6 + (4) - When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used in - calculations when the day of the week and the year are specified. + For a naive object, the ``%z`` and ``%Z`` format codes are replaced by empty + strings. + + For an aware object: + + ``%z`` + :meth:`utcoffset` is transformed into a 5-character string of the form + +HHMM or -HHMM, where HH is a 2-digit string giving the number of UTC + offset hours, and MM is a 2-digit string giving the number of UTC offset + minutes. For example, if :meth:`utcoffset` returns + ``timedelta(hours=-3, minutes=-30)``, ``%z`` is replaced with the string + ``'-0330'``. + + ``%Z`` + If :meth:`tzname` returns ``None``, ``%Z`` is replaced by an empty + string. Otherwise ``%Z`` is replaced by the returned value, which must + be a string. (5) - For example, if :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, - ``%z`` is replaced with the string ``'-0330'``. + When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used + in calculations when the day of the week and the year are specified. .. rubric:: Footnotes From 8237f93f2219dc0dd1d6cabbe04613de0d3c5d06 Mon Sep 17 00:00:00 2001 From: David Wolever Date: Sat, 13 Apr 2013 20:50:24 -0400 Subject: [PATCH 045/480] Adding localized examples to strftime documentation. --- Doc/library/datetime.rst | 86 +++++++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 28 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index d5c62af43dc..a60fa38dc38 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1616,11 +1616,19 @@ platforms. Regardless of platform, years before 1900 cannot be used. +-----------+--------------------------------+------------------------+-------+ | Directive | Meaning | Example | Notes | +===========+================================+========================+=======+ -| ``%a`` | Weekday as locale's | Sun, Mon, ..., Sat | | -| | abbreviated name. | | | +| ``%a`` | Weekday as locale's || Sun, Mon, ..., Sat | \(1) | +| | abbreviated name. | (en_US); | | +| | || So, Mo, ..., Sa | | +| | | (de_DE); | | +| | || æ—¥, 月, ..., 土 | | +| | | (ja_JP) | | +-----------+--------------------------------+------------------------+-------+ -| ``%A`` | Weekday as locale's full name. | Sunday, Monday, ..., | | -| | | Saturday | | +| ``%A`` | Weekday as locale's full name. || Sunday, Monday, ..., | \(1) | +| | | Saturday (en_US); | | +| | || Sonntag, Montag, ..., | | +| | | Samstag (de_DE); | | +| | || 日曜日, 月曜日, ..., | | +| | | 土曜日 (ja_JP) | | +-----------+--------------------------------+------------------------+-------+ | ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | | | where 0 is Sunday and 6 is | | | @@ -1629,14 +1637,18 @@ platforms. Regardless of platform, years before 1900 cannot be used. | ``%d`` | Day of the month as a | 01, 02, ..., 31 | | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%j`` | Day of the year as a | 001, 002, ..., 366 | | -| | zero-padded decimal number. | | | +| ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | +| | name. | (en_US); | | +| | || Jan, Feb, ..., Dez | | +| | | (de_DE); | | +| | || 1, 2, ..., 12 (ja_JP) | | +-----------+--------------------------------+------------------------+-------+ -| ``%b`` | Month as locale's abbreviated | Jan, Feb, ..., Dec | | -| | name. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%B`` | Month as locale's full name. | January, February, | | -| | | ..., December | | +| ``%B`` | Month as locale's full name. || January, February, | \(1) | +| | | ..., December (en_US);| | +| | || Januar, Februar, ..., | | +| | | Dezember (de_DE); | | +| | || 1月, 2月, ..., 12月 | | +| | | (ja_JP) | | +-----------+--------------------------------+------------------------+-------+ | ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | | | | decimal number. | | | @@ -1653,27 +1665,31 @@ platforms. Regardless of platform, years before 1900 cannot be used. | ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%p`` | Locale's equivalent of either | AM, PM | \(1) | -| | AM or PM. | | | +| ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | +| | AM or PM. || am, pm (de_DE); | \(2) | +| | || AM, PM (ja_JP) | | +-----------+--------------------------------+------------------------+-------+ | ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 61 | \(2) | +| ``%S`` | Second as a zero-padded | 00, 01, ..., 61 | \(3) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(3) | +| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(4) | | | number, zero-padded on the | 999999 | | | | left. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%z`` | UTC offset in the form +HHMM | (empty), +0000, -0400, | \(4) | +| ``%z`` | UTC offset in the form +HHMM | (empty), +0000, -0400, | \(5) | | | or -HHMM (empty string if the | +1030 | | | | the object is naive). | | | +-----------+--------------------------------+------------------------+-------+ | ``%Z`` | Time zone name (empty string | (empty), UTC, EST, CST | | | | if the object is naive). | | | +-----------+--------------------------------+------------------------+-------+ -| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(5) | +| ``%j`` | Day of the year as a | 001, 002, ..., 366 | | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(6) | | | (Sunday as the first day of | | | | | the week) as a zero padded | | | | | decimal number. All days in a | | | @@ -1681,7 +1697,7 @@ platforms. Regardless of platform, years before 1900 cannot be used. | | Sunday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(5) | +| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(6) | | | (Monday as the first day of | | | | | the week) as a decimal number. | | | | | All days in a new year | | | @@ -1689,14 +1705,21 @@ platforms. Regardless of platform, years before 1900 cannot be used. | | are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%c`` | Locale's appropriate date and | Mon Aug 1 16:00:00 | | -| | time representation. | 1988 | | +| ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | +| | time representation. | 1988 (en_US), | | +| | || Di 16 Aug 21:30:00 | | +| | | 1988 (de_DE), | | +| | || ç« 8/16 21:30:00 | | +| | | 1988 (ja_JP) | | +-----------+--------------------------------+------------------------+-------+ -| ``%x`` | Locale's appropriate date | 08/16/88 | | -| | representation. | | | +| ``%x`` | Locale's appropriate date || 08/16/88 (None), | \(1) | +| | representation. || 08/16/1988 (en_US), | | +| | || 16.08.1988 (de_DE), | | +| | || 1988/08/16 (ja_JP) | | +-----------+--------------------------------+------------------------+-------+ -| ``%X`` | Locale's appropriate time | 16:00:00 | | -| | representation. | | | +| ``%X`` | Locale's appropriate time || 21:30:00 (en_US), | \(1) | +| | representation. || 21:30:00 (de_DE), | | +| | || 21時30分00秒 (ja_JP) | | +-----------+--------------------------------+------------------------+-------+ | ``%%`` | A literal ``'%'`` character. | % | | +-----------+--------------------------------+------------------------+-------+ @@ -1704,10 +1727,17 @@ platforms. Regardless of platform, years before 1900 cannot be used. Notes: (1) + Because the format depends on the current locale, care should be taken when + making assumptions about the output value. Field orderings will vary (for + example, "month/day/year" versus "day/month/year"), and the output may + contain UTF-8 encoded unicode characters (for example, the ``ja_JP`` locale + may include Japanese characters). + +(2) When used with the :meth:`strptime` method, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. -(2) +(3) The range really is ``0`` to ``61``; according to the Posix standard this accounts for leap seconds and the (very rare) double leap seconds. The :mod:`time` module may produce and does accept leap seconds since @@ -1715,7 +1745,7 @@ Notes: does not accept leap seconds in :meth:`strptime` input nor will it produce them in :func:`strftime` output. -(3) +(4) ``%f`` is an extension to the set of format characters in the C standard (but implemented separately in datetime objects, and therefore always available). When used with the :meth:`strptime` method, the ``%f`` @@ -1723,7 +1753,7 @@ Notes: .. versionadded:: 2.6 -(4) +(5) For a naive object, the ``%z`` and ``%Z`` format codes are replaced by empty strings. @@ -1742,7 +1772,7 @@ Notes: string. Otherwise ``%Z`` is replaced by the returned value, which must be a string. -(5) +(6) When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used in calculations when the day of the week and the year are specified. From e4d27d20f45f14f620da96356926faec1e5984b0 Mon Sep 17 00:00:00 2001 From: David Wolever Date: Sat, 13 Apr 2013 22:40:11 -0400 Subject: [PATCH 046/480] Correct misleading strftime documentation. strftime using locale-aware formatting directives will often, but not awlays, produce UTF-8-encoded Unicode. --- Doc/library/datetime.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index a60fa38dc38..cf41d434c0e 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1730,8 +1730,10 @@ Notes: Because the format depends on the current locale, care should be taken when making assumptions about the output value. Field orderings will vary (for example, "month/day/year" versus "day/month/year"), and the output may - contain UTF-8 encoded unicode characters (for example, the ``ja_JP`` locale - may include Japanese characters). + contain Unicode characters (encoded, by default, using UTF-8, but this may + vary based on the locale; for example, the ``ja_JP`` locale contains UTF-8 + encoded Japanese characters, but ``ja_JP.SJIS`` contains Shift JIS encoded + Japanese characters). (2) When used with the :meth:`strptime` method, the ``%p`` directive only affects From f8feec8ead73208c7dd4bba72bcb3ced23e8c770 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 14 Apr 2013 10:13:42 +0200 Subject: [PATCH 047/480] Clarify point in name mangling doc. --- Doc/reference/expressions.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index a11ee6e2624..4b05c374f18 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -96,14 +96,13 @@ exception. definition begins with two or more underscore characters and does not end in two or more underscores, it is considered a :dfn:`private name` of that class. Private names are transformed to a longer form before code is generated for -them. The transformation inserts the class name in front of the name, with -leading underscores removed, and a single underscore inserted in front of the -class name. For example, the identifier ``__spam`` occurring in a class named -``Ham`` will be transformed to ``_Ham__spam``. This transformation is -independent of the syntactical context in which the identifier is used. If the -transformed name is extremely long (longer than 255 characters), implementation -defined truncation may happen. If the class name consists only of underscores, -no transformation is done. +them. The transformation inserts the class name, with leading underscores +removed and a single underscore inserted, in front of the name. For example, +the identifier ``__spam`` occurring in a class named ``Ham`` will be transformed +to ``_Ham__spam``. This transformation is independent of the syntactical +context in which the identifier is used. If the transformed name is extremely +long (longer than 255 characters), implementation defined truncation may happen. +If the class name consists only of underscores, no transformation is done. From 5ed1500b58de185dd1876c268b0337da03cf7b91 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 14 Apr 2013 10:17:35 +0200 Subject: [PATCH 048/480] Update using/mac documentation which was still on 2.5. --- Doc/using/mac.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 7dcc1b23f9c..5f29812e448 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -25,14 +25,14 @@ installers for the latest 2.3 release for Mac OS 9 and related documentation. Getting and Installing MacPython ================================ -Mac OS X 10.5 comes with Python 2.5.1 pre-installed by Apple. If you wish, you +Mac OS X 10.8 comes with Python 2.7 pre-installed by Apple. If you wish, you are invited to install the most recent version of Python from the Python website (http://www.python.org). A current "universal binary" build of Python, which runs natively on the Mac's new Intel and legacy PPC CPU's, is available there. What you get after installing is a number of things: -* A :file:`MacPython 2.5` folder in your :file:`Applications` folder. In here +* A :file:`MacPython 2.7` folder in your :file:`Applications` folder. In here you find IDLE, the development environment that is a standard part of official Python distributions; PythonLauncher, which handles double-clicking Python scripts from the Finder; and the "Build Applet" tool, which allows you to @@ -100,7 +100,7 @@ aware of: programs that talk to the Aqua window manager (in other words, anything that has a GUI) need to be run in a special way. Use :program:`pythonw` instead of :program:`python` to start such scripts. -With Python 2.5, you can use either :program:`python` or :program:`pythonw`. +With Python 2.7, you can use either :program:`python` or :program:`pythonw`. Configuration @@ -133,13 +133,11 @@ Installing Additional Python Packages There are several methods to install additional Python packages: -* http://pythonmac.org/packages/ contains selected compiled packages for Python - 2.5, 2.4, and 2.3. - * Packages can be installed via the standard Python distutils mode (``python setup.py install``). -* Many packages can also be installed via the :program:`setuptools` extension. +* Many packages can also be installed via the :program:`setuptools` extension + or :program:`pip` wrapper, see http://www.pip-installer.org/. GUI Programming on the Mac @@ -167,7 +165,7 @@ http://www.riverbankcomputing.co.uk/software/pyqt/intro. Distributing Python Applications on the Mac =========================================== -The "Build Applet" tool that is placed in the MacPython 2.5 folder is fine for +The "Build Applet" tool that is placed in the MacPython 2.7 folder is fine for packaging small Python scripts on your own machine to run as a standard Mac application. This tool, however, is not robust enough to distribute Python applications to other users. From 20b800fdc847985cd881eee253daa59ab30730f9 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 14 Apr 2013 10:31:06 +0200 Subject: [PATCH 049/480] Closes #17726: small clarification in design FAQ. --- Doc/faq/design.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 833e58666b8..73c90af0372 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -910,8 +910,8 @@ There are several reasons to allow this. When you have a literal value for a list, tuple, or dictionary spread across multiple lines, it's easier to add more elements because you don't have to -remember to add a comma to the previous line. The lines can also be sorted in -your editor without creating a syntax error. +remember to add a comma to the previous line. The lines can also be reordered +without creating a syntax error. Accidentally omitting the comma can lead to errors that are hard to diagnose. For example:: From 156c9b9719b5fb8c309b613f63256e473e8757cd Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 14 Apr 2013 10:36:03 +0200 Subject: [PATCH 050/480] Closes #17661: fix references to repr() going to module repr. --- Doc/library/stdtypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 5d6d35343e0..1292e9bfd41 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -26,7 +26,7 @@ instances and exceptions. Some operations are supported by several object types; in particular, practically all objects can be compared, tested for truth value, and converted -to a string (with the :func:`repr` function or the slightly different +to a string (with the :ref:`repr() ` function or the slightly different :func:`str` function). The latter function is implicitly used when an object is written by the :func:`print` function. @@ -1452,7 +1452,7 @@ The conversion types are: | | character string). | | +------------+-----------------------------------------------------+-------+ | ``'r'`` | String (converts any Python object using | \(5) | -| | :func:`repr`). | | +| | :ref:`repr() `). | | +------------+-----------------------------------------------------+-------+ | ``'s'`` | String (converts any Python object using | \(6) | | | :func:`str`). | | From 392223bff0525db0630519bd86a4cd6b1cb0b9fe Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 14 Apr 2013 10:59:04 +0200 Subject: [PATCH 051/480] bind('') does not do the same as bind('localhost') --- Doc/howto/sockets.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index c4b3f71af75..6338d3b846e 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -88,9 +88,11 @@ creates a "server socket":: serversocket.listen(5) A couple things to notice: we used ``socket.gethostname()`` so that the socket -would be visible to the outside world. If we had used ``s.bind(('', 80))`` or -``s.bind(('localhost', 80))`` or ``s.bind(('127.0.0.1', 80))`` we would still -have a "server" socket, but one that was only visible within the same machine. +would be visible to the outside world. If we had used ``s.bind(('localhost', +80))`` or ``s.bind(('127.0.0.1', 80))`` we would still have a "server" socket, +but one that was only visible within the same machine. ``s.bind(('', 80))`` +specifies that the socket is reachable by any address the machine happens to +have. A second thing to note: low number ports are usually reserved for "well known" services (HTTP, SNMP etc). If you're playing around, use a nice high number (4 From 33bf855c08c888986afb491474dd8572630e4b6c Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 14 Apr 2013 11:12:16 +0200 Subject: [PATCH 052/480] Closes #13638: document PyErr_SetFromErrnoWithFilenameObject, PyErr_SetFromWindowsErrWithFilenameObject, and PyErr_SetExcFromWindowsErrWithFilenameObject. Note that PyErr_SetExcFromWindowsErrWithFilenameObjectAndSomeOtherParametersSoThatTheNameGetsSoLongThatNobodyIsEverGonnaUseThisStrangeFunctionForAnything is still undocumented. --- Doc/c-api/exceptions.rst | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 025b75a70b2..91964d05737 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -192,12 +192,19 @@ is a separate error indicator for each thread. when the system call returns an error. -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if - *filename* is not *NULL*, it is passed to the constructor of *type* as a third - parameter. In the case of exceptions such as :exc:`IOError` and :exc:`OSError`, - this is used to define the :attr:`filename` attribute of the exception instance. + *filenameObject* is not *NULL*, it is passed to the constructor of *type* as + a third parameter. In the case of exceptions such as :exc:`IOError` and + :exc:`OSError`, this is used to define the :attr:`filename` attribute of the + exception instance. + + +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) + + Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename + is given as a C string. .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) @@ -220,14 +227,29 @@ is a separate error indicator for each thread. .. versionadded:: 2.3 -.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) +.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilenameObject(int ierr, PyObject *filenameObject) Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior that - if *filename* is not *NULL*, it is passed to the constructor of + if *filenameObject* is not *NULL*, it is passed to the constructor of :exc:`WindowsError` as a third parameter. Availability: Windows. -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, char *filename) +.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the + filename is given as a C string. Availability: Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an + additional parameter specifying the exception type to be raised. + Availability: Windows. + + .. versionadded:: 2.3 + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename) Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional parameter specifying the exception type to be raised. Availability: Windows. From 7eb94d7af6418153b5fbcb7abcc7a873b81367b6 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 14 Apr 2013 11:45:16 +0200 Subject: [PATCH 053/480] Fix refcount leak in the reference counting example section (!). --- Doc/c-api/intro.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index c3a9ed633bd..6414277c849 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -255,8 +255,10 @@ sets all items of a list (actually, any mutable sequence) to a given item:: PyObject *index = PyInt_FromLong(i); if (!index) return -1; - if (PyObject_SetItem(target, index, item) < 0) + if (PyObject_SetItem(target, index, item) < 0) { + Py_DECREF(index); return -1; + } Py_DECREF(index); } return 0; From df5c7472c18826fbd55b44c47a7b11bb37028de9 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 14 Apr 2013 11:47:46 +0200 Subject: [PATCH 054/480] Clarify that the function *definition* creates the function and the binding in the module globals. --- Doc/tutorial/modules.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index d1a72c082bf..f5d285ec312 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -550,6 +550,6 @@ modules found in a package. .. rubric:: Footnotes .. [#] In fact function definitions are also 'statements' that are 'executed'; the - execution of a module-level function enters the function name in the module's - global symbol table. + execution of a module-level function definition enters the function name in + the module's global symbol table. From 07a5f0005e8d43196debdaab4581c1ac007869a6 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 14 Apr 2013 11:53:36 +0200 Subject: [PATCH 055/480] The im_class of unbound class methods is actually not the same as im_self. --- Doc/reference/datamodel.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 79114c1ec25..aa0a5871822 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -623,9 +623,8 @@ Callable types single: im_self (method attribute) When a user-defined method object is created by retrieving a class method object - from a class or instance, its :attr:`im_self` attribute is the class itself (the - same as the :attr:`im_class` attribute), and its :attr:`im_func` attribute is - the function object underlying the class method. + from a class or instance, its :attr:`im_self` attribute is the class itself, and + its :attr:`im_func` attribute is the function object underlying the class method. When an unbound user-defined method object is called, the underlying function (:attr:`im_func`) is called, with the restriction that the first argument must @@ -797,8 +796,8 @@ Classes associated class is either :class:`C` or one of its base classes, it is transformed into an unbound user-defined method object whose :attr:`im_class` attribute is :class:`C`. When it would yield a class method object, it is - transformed into a bound user-defined method object whose :attr:`im_class` - and :attr:`im_self` attributes are both :class:`C`. When it would yield a + transformed into a bound user-defined method object whose + :attr:`im_self` attribute is :class:`C`. When it would yield a static method object, it is transformed into the object wrapped by the static method object. See section :ref:`descriptors` for another way in which attributes retrieved from a class may differ from those actually contained in From 30f80448db6d1f0849b19049e86943dc07de88c7 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 14 Apr 2013 11:58:54 +0200 Subject: [PATCH 056/480] The Integral class does not contain implementations for the bit-shifting operations. (See #3056.) --- Doc/library/numbers.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index f46e8ac824e..8811b5df8ce 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -73,10 +73,10 @@ The numeric tower .. class:: Integral - Subtypes :class:`Rational` and adds a conversion to :class:`int`. - Provides defaults for :func:`float`, :attr:`~Rational.numerator`, and - :attr:`~Rational.denominator`, and bit-string operations: ``<<``, - ``>>``, ``&``, ``^``, ``|``, ``~``. + Subtypes :class:`Rational` and adds a conversion to :class:`int`. Provides + defaults for :func:`float`, :attr:`~Rational.numerator`, and + :attr:`~Rational.denominator`. Adds abstract methods for ``**`` and + bit-string operations: ``<<``, ``>>``, ``&``, ``^``, ``|``, ``~``. Notes for type implementors From c218b4497ac622bc071cff678125dc4475774629 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 14 Apr 2013 12:02:43 +0200 Subject: [PATCH 057/480] Fix example ignoring ".svn" directories in compileall. --- Doc/library/compileall.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index cf0d5f85a0a..0e79a5ab5d6 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -127,7 +127,7 @@ subdirectory and all its subdirectories:: # Perform same compilation, excluding files in .svn directories. import re - compileall.compile_dir('Lib/', rx=re.compile('/[.]svn'), force=True) + compileall.compile_dir('Lib/', rx=re.compile(r'[/\\][.]svn'), force=True) .. seealso:: From 56e58506e376d6024ffc8d51f55d0faaac0a6f35 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 14 Apr 2013 18:52:15 +0300 Subject: [PATCH 058/480] Issue #17221: Resort Misc/NEWS. --- Misc/NEWS | 293 +++++++++++++++++++++++++++++------------------------- 1 file changed, 155 insertions(+), 138 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 1dd72fbbd83..eda841fd992 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -70,7 +70,6 @@ IDLE - Issue #6649: Fixed missing exit status in IDLE. Patch by Guilherme Polo. - Documentation ------------- @@ -97,11 +96,14 @@ Core and Builtins mapping such that any type with a __getitem__ can be used on the right hand side. -Library -------- +IDLE +---- - Issue #17625: In IDLE, close the replace dialog after it is used. +Tests +----- + - Issue #17531: Fix tests that thought group and user ids were always the int type. Also, always allow -1 as a valid group and user id. @@ -237,8 +239,6 @@ Core and Builtins - Issue #15604: Update uses of PyObject_IsTrue() to check for and handle errors correctly. Patch by Serhiy Storchaka. -- Issue #15041: Update "see also" list in tkinter documentation. - - Issue #14579: Fix error handling bug in the utf-16 decoder. Patch by Serhiy Storchaka. @@ -330,7 +330,7 @@ Library - Issue #12718: Fix interaction with winpdb overriding __import__ by setting importer attribute on BaseConfigurator instance. - + - Issue #17521: Corrected non-enabling of logger following two calls to fileConfig(). @@ -403,14 +403,9 @@ Library - Issue #6975: os.path.realpath() now correctly resolves multiple nested symlinks on POSIX platforms. -- Issue #17156: pygettext.py now correctly escapes non-ascii characters. - - Issue #7358: cStringIO.StringIO now supports writing to and reading from a stream larger than 2 GiB on 64-bit systems. -- IDLE was displaying spurious SystemExit tracebacks when running scripts - that terminated by raising SystemExit (i.e. unittest and turtledemo). - - Issue #10355: In SpooledTemporaryFile class mode and name properties and xreadlines method now work for unrolled files. encoding and newlines properties now removed as they have no sense and always produced @@ -462,9 +457,6 @@ Library - Issue #17051: Fix a memory leak in os.path.isdir() on Windows. Patch by Robert Xiao. -- Issue #9290: In IDLE the sys.std* streams now implement io.TextIOBase - interface and support all mandatory methods and properties. - - Issue #13454: Fix a crash when deleting an iterator created by itertools.tee() if all other iterators were very advanced before. @@ -482,9 +474,6 @@ Library - Issue #9720: zipfile now writes correct local headers for files larger than 4 GiB. -- Issue #16829: IDLE printing no longer fails if there are spaces or other - special characters in the file path. - - Issue #13899: \A, \Z, and \B now correctly match the A, Z, and B literals when used inside character classes (e.g. '[\A]'). Patch by Matthew Barnett. @@ -502,8 +491,6 @@ Library - Issue #16828: Fix error incorrectly raised by bz2.compress(''). Patch by Martin Packman. -- Issue #16819: IDLE method completion now correctly works for unicode literals. - - Issue #9586: Redefine SEM_FAILED on MacOSX to keep compiler happy. - Issue #10527: make multiprocessing use poll() instead of select() if available. @@ -514,12 +501,6 @@ Library - Issue #12065: connect_ex() on an SSL socket now returns the original errno when the socket's timeout expires (it used to return None). -- Issue #16504: IDLE now catches SyntaxErrors raised by tokenizer. Patch by - Roger Serwy. - -- Issue #16702: test_urllib2_localnet tests now correctly ignores proxies for - localhost tests. - - Issue #16713: Fix the parsing of tel url with params using urlparse module. - Issue #16443: Add docstrings to regular expression match objects. @@ -558,8 +539,6 @@ Library list() calls aren't added to filter(), map(), and zip() which are directly passed enumerate(). -- Issue #16476: Fix json.tool to avoid including trailing whitespace. - - Issue #1160: Fix compiling large regular expressions on UCS2 builds. Patch by Serhiy Storchaka. @@ -590,9 +569,6 @@ Library - Issue #16152: fix tokenize to ignore whitespace at the end of the code when no newline is found. Patch by Ned Batchelder. -- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu - Patch by Todd Rovito. - - Issue #16230: Fix a crash in select.select() when one the lists changes size while iterated on. Patch by Serhiy Storchaka. @@ -678,15 +654,9 @@ Library - Issue #15424: Add a __sizeof__ implementation for array objects. Patch by Ludwig Hähne. -- Issue #13052: Fix IDLE crashing when replace string in Search/Replace dialog - ended with '\'. Patch by Roger Serwy. - - Issue #15538: Fix compilation of the getnameinfo() / getaddrinfo() emulation code. Patch by Philipp Hagemeister. -- Issue #9803: Don't close IDLE on saving if breakpoint is open. - Patch by Roger Serwy. - - Issue #12288: Consider '0' and '0.0' as valid initialvalue for tkinter SimpleDialog. @@ -761,23 +731,6 @@ Library - Issue #12157: Make pool.map() empty iterables correctly. Initial patch by mouad. -- Issue #14958: Change IDLE systax highlighting to recognize all string and byte - literals currently supported in Python 2.7. - -- Issue #14962: Update text coloring in IDLE shell window after changing - options. Patch by Roger Serwy. - -- Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. - -- Issue #12510: Attempting to get invalid tooltip no longer closes Idle. - Original patch by Roger Serwy. - -- Issue #10365: File open dialog now works instead of crashing - even when parent window is closed. Patch by Roger Serwy. - -- Issue #14876: Use user-selected font for highlight configuration. - Patch by Roger Serwy. - - Issue #14036: Add an additional check to validate that port in urlparse does not go in illegal range and returns None. @@ -867,11 +820,6 @@ Library returned. This avoids crashing the server loop when a signal is received. Patch by Jerzy Kozera. -- Issue #14409: IDLE now properly executes commands in the Shell window - when it cannot read the normal config files on startup and - has to use the built-in default key bindings. - There was previously a bug in one of the defaults. - - Issue #10340: asyncore - properly handle EINVAL in dispatcher constructor on OSX; avoid to call handle_connect in case of a disconnected socket which was not meant to connect. @@ -879,9 +827,6 @@ Library - Issue #12757: Fix the skipping of doctests when python is run with -OO so that it works in unittest's verbose mode as well as non-verbose mode. -- Issue #3573: IDLE hangs when passing invalid command line args - (directory(ies) instead of file(s)) (Patch by Guilherme Polo) - - Issue #13694: asynchronous connect in asyncore.dispatcher does not set addr attribute. @@ -889,8 +834,6 @@ Library - Issue #11199: Fix the with urllib which hangs on particular ftp urls. -- Issue #5219: Prevent event handler cascade in IDLE. - - Issue #14252: Fix subprocess.Popen.terminate() to not raise an error under Windows when the child process has already exited. @@ -904,9 +847,6 @@ Library - Issue #2945: Make the distutils upload command aware of bdist_rpm products. -- Issue #13447: Add a test file to host regression tests for bugs in the - scripts found in the Tools directory. - - Issue #6884: Fix long-standing bugs with MANIFEST.in parsing in distutils on Windows. @@ -958,9 +898,68 @@ Extension Modules and problematic Apple llvm-gcc compiler. If original compiler is not available, use clang instead by default. +IDLE +---- + +- IDLE was displaying spurious SystemExit tracebacks when running scripts + that terminated by raising SystemExit (i.e. unittest and turtledemo). + +- Issue #9290: In IDLE the sys.std* streams now implement io.TextIOBase + interface and support all mandatory methods and properties. + +- Issue #16829: IDLE printing no longer fails if there are spaces or other + special characters in the file path. + +- Issue #16819: IDLE method completion now correctly works for unicode literals. + +- Issue #16504: IDLE now catches SyntaxErrors raised by tokenizer. Patch by + Roger Serwy. + +- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu + Patch by Todd Rovito. + +- Issue #13052: Fix IDLE crashing when replace string in Search/Replace dialog + ended with '\'. Patch by Roger Serwy. + +- Issue #9803: Don't close IDLE on saving if breakpoint is open. + Patch by Roger Serwy. + +- Issue #14958: Change IDLE systax highlighting to recognize all string and byte + literals currently supported in Python 2.7. + +- Issue #14962: Update text coloring in IDLE shell window after changing + options. Patch by Roger Serwy. + +- Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. + +- Issue #12510: Attempting to get invalid tooltip no longer closes IDLE. + Original patch by Roger Serwy. + +- Issue #10365: File open dialog now works instead of crashing + even when parent window is closed. Patch by Roger Serwy. + +- Issue #14876: Use user-selected font for highlight configuration. + Patch by Roger Serwy. + +- Issue #14409: IDLE now properly executes commands in the Shell window + when it cannot read the normal config files on startup and + has to use the built-in default key bindings. + There was previously a bug in one of the defaults. + +- Issue #3573: IDLE hangs when passing invalid command line args + (directory(ies) instead of file(s)) (Patch by Guilherme Polo) + +- Issue #5219: Prevent event handler cascade in IDLE. + Tests ----- +- Issue #16702: test_urllib2_localnet tests now correctly ignores proxies for + localhost tests. + +- Issue #13447: Add a test file to host regression tests for bugs in the + scripts found in the Tools directory. + - Issue #11420: make test suite pass with -B/DONTWRITEBYTECODE set. Initial patch by Thomas Wouters. @@ -1090,17 +1089,23 @@ Build Tools/Demos ----------- +- Issue #17156: pygettext.py now correctly escapes non-ascii characters. + - Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py. Now pindent.py works with a "with" statement. pindent.py no longer produces improper indentation. pindent.py now works with continued lines broken after "class" or "def" keywords and with continuations at the start of line. +- Issue #16476: Fix json.tool to avoid including trailing whitespace. + - Issue #13301: use ast.literal_eval() instead of eval() in Tools/i18n/msgfmt.py Patch by Serhiy Storchaka. Documentation ------------- +- Issue #15041: Update "see also" list in tkinter documentation. + - Issue #17412: update 2.7 Doc/make.bat to also use sphinx-1.0.7. - Issue #17047: remove doubled words in docs and docstrings @@ -1343,21 +1348,8 @@ Library - Issue #10811: Fix recursive usage of cursors. Instead of crashing, raise a ProgrammingError now. -- Issue #10881: Fix test_site failures with OS X framework builds. - -- Issue #964437 Make IDLE help window non-modal. - Patch by Guilherme Polo and Roger Serwy. - -- Issue #13933: IDLE auto-complete did not work with some imported - module, like hashlib. (Patch by Roger Serwy) - -- Issue #13901: Prevent test_distutils failures on OS X with --enable-shared. - - Issue #13676: Handle strings with embedded zeros correctly in sqlite3. -- Issue #13506: Add '' to path for IDLE Shell when started and restarted with Restart Shell. - Original patches by Marco Scataglini and Roger Serwy. - - Issue #13806: The size check in audioop decompression functions was too strict and could reject valid compressed data. Patch by Oleg Plakhotnyuk. @@ -1396,10 +1388,6 @@ Library - Issue #8035: urllib: Fix a bug where the client could remain stuck after a redirection or an error. -- Issue #4625: If IDLE cannot write to its recent file or breakpoint - files, display a message popup and continue rather than crash. - (original patch by Roger Serwy) - - tarfile.py: Correctly detect bzip2 compressed streams with blocksizes other than 900k. @@ -1429,9 +1417,6 @@ Library node when it is the only child of an element. Initial patch by Dan Kenigsberg. -- Issue #8793: Prevent IDLE crash when given strings with invalid hex escape - sequences. - - Issues #1745761, #755670, #13357, #12629, #1200313: HTMLParser now correctly handles non-valid attributes, including adjacent and unquoted attributes. @@ -1454,9 +1439,6 @@ Library - Issue #10817: Fix urlretrieve function to raise ContentTooShortError even when reporthook is None. Patch by Jyrki Pulliainen. -- Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart. - (Patch by Roger Serwy) - - Issue #7334: close source files on ElementTree.parse and iterparse. - Issue #13232: logging: Improved logging of exceptions in the presence of @@ -1701,6 +1683,28 @@ Extension Modules signature. Without this, architectures where sizeof void* != sizeof int are broken. Patch given by Hallvard B Furuseth. +IDLE +---- + +- Issue #964437 Make IDLE help window non-modal. + Patch by Guilherme Polo and Roger Serwy. + +- Issue #13933: IDLE auto-complete did not work with some imported + module, like hashlib. (Patch by Roger Serwy) + +- Issue #13506: Add '' to path for IDLE Shell when started and restarted with Restart Shell. + Original patches by Marco Scataglini and Roger Serwy. + +- Issue #4625: If IDLE cannot write to its recent file or breakpoint + files, display a message popup and continue rather than crash. + (original patch by Roger Serwy) + +- Issue #8793: Prevent IDLE crash when given strings with invalid hex escape + sequences. + +- Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart. + (Patch by Roger Serwy) + Build ----- @@ -1741,6 +1745,10 @@ Tests - Issue #11689: Fix a variable scoping error in an sqlite3 test. Initial patch by Torsten Landschoff. +- Issue #10881: Fix test_site failures with OS X framework builds. + +- Issue #13901: Prevent test_distutils failures on OS X with --enable-shared. + - Issue #13304: Skip test case if user site-packages disabled (-s or PYTHONNOUSERSITE). (Patch by Carl Meyer) @@ -1913,9 +1921,6 @@ Core and Builtins Library ------- -- Issue #12590: IDLE editor window now always displays the first line - when opening a long file. With Tk 8.5, the first line was hidden. - - Issue #12161: Cause StringIO.getvalue() to raise a ValueError when used on a closed StringIO instance. @@ -1937,9 +1942,6 @@ Library - Issue #12124: zipimport doesn't keep a reference to zlib.decompress() anymore to be able to unload the module. -- Issue #11088: don't crash when using F5 to run a script in IDLE on MacOSX - with Tk 8.5. - - Issue #10154, #10090: change the normalization of UTF-8 to "UTF-8" instead of "UTF8" in the locale module as the latter is not supported MacOSX and OpenBSD. @@ -1959,8 +1961,6 @@ Library - Issue #12012: ssl.PROTOCOL_SSLv2 becomes optional. -- Issue #11164: Remove obsolete allnodes test from minidom test. - - Issue #11927: SMTP_SSL now uses port 465 by default as documented. Patch by Kasun Herath. @@ -2112,17 +2112,6 @@ Library - Issue #8275: Fix passing of callback arguments with ctypes under Win64. Patch by Stan Mihai. -- Issue #10940: Workaround an IDLE hang on Mac OS X 10.6 when using the - menu accelerators for Open Module, Go to Line, and New Indent Width. - The accelerators still work but no longer appear in the menu items. - -- Issue #10907: Warn OS X 10.6 IDLE users to use ActiveState Tcl/Tk 8.5, rather - than the currently problematic Apple-supplied one, when running with the - 64-/32-bit installer variant. - -- Issue #11052: Correct IDLE menu accelerators on Mac OS X for Save - commands. - - Issue #10949: Improved robustness of rotating file handlers. - Issue #10955: Fix a potential crash when trying to mmap() a file past its @@ -2131,9 +2120,6 @@ Library - Issue #10898: Allow compiling the posix module when the C library defines a symbol named FSTAT. -- Issue #6075: IDLE on Mac OS X now works with both Carbon AquaTk and - Cocoa AquaTk. - - Issue #10916: mmap should not segfault when a file is mapped using 0 as length and a non-zero offset, and an attempt to read past the end of file is made (IndexError is raised instead). Patch by Ross Lagerwall. @@ -2192,8 +2178,6 @@ Library - Issue #6791: Limit header line length (to 65535 bytes) in http.client, to avoid denial of services from the other party. -- Issue #10404: Use ctl-button-1 on OSX for the context menu in Idle. - - Issue #9907: Fix tab handling on OSX when using editline by calling rl_initialize first, then setting our custom defaults, then reading .editrc. @@ -2211,11 +2195,6 @@ Library - Issue #10695: passing the port as a string value to telnetlib no longer causes debug mode to fail. -- Issue #10107: Warn about unsaved files in IDLE on OSX. - -- Issue #10406: Enable Rstrip IDLE extension on OSX (just like on other - platforms). - - Issue #10478: Reentrant calls inside buffered IO objects (for example by way of a signal handler) now raise a RuntimeError instead of freezing the current process. @@ -2262,6 +2241,39 @@ Extension Modules - Issue #678250: Make mmap flush a noop on ACCESS_READ and ACCESS_COPY. +IDLE +---- + +- Issue #11718: IDLE's open module dialog couldn't find the __init__.py + file in a package. + +- Issue #12590: IDLE editor window now always displays the first line + when opening a long file. With Tk 8.5, the first line was hidden. + +- Issue #11088: don't crash when using F5 to run a script in IDLE on MacOSX + with Tk 8.5. + +- Issue #10940: Workaround an IDLE hang on Mac OS X 10.6 when using the + menu accelerators for Open Module, Go to Line, and New Indent Width. + The accelerators still work but no longer appear in the menu items. + +- Issue #10907: Warn OS X 10.6 IDLE users to use ActiveState Tcl/Tk 8.5, rather + than the currently problematic Apple-supplied one, when running with the + 64-/32-bit installer variant. + +- Issue #11052: Correct IDLE menu accelerators on Mac OS X for Save + commands. + +- Issue #6075: IDLE on Mac OS X now works with both Carbon AquaTk and + Cocoa AquaTk. + +- Issue #10404: Use ctl-button-1 on OSX for the context menu in Idle. + +- Issue #10107: Warn about unsaved files in IDLE on OSX. + +- Issue #10406: Enable Rstrip IDLE extension on OSX (just like on other + platforms). + Build ----- @@ -2307,15 +2319,11 @@ Build - Issue #1099: Fix the build on MacOSX when building a framework with pydebug using GCC 4.0. -IDLE ----- - -- Issue #11718: IDLE's open module dialog couldn't find the __init__.py - file in a package. - Tests ----- +- Issue #11164: Remove obsolete allnodes test from minidom test. + - Issue #12205: Fix test_subprocess failure due to uninstalled test data. - Issue #5723: Improve json tests to be executed with and without accelerations. @@ -2384,9 +2392,6 @@ Library - Issue #4493: urllib2 adds '/' in front of path components which does not start with '/. Common behavior exhibited by browsers and other clients. -- Issue #6378: idle.bat now runs with the appropriate Python version rather than - the system default. Patch by Sridhar Ratnakumar. - - Issue #10407: Fix one NameError in distutils. - Issue #10198: fix duplicate header written to wave files when writeframes() @@ -2397,6 +2402,12 @@ Library - Issue #5111: IPv6 Host in the Header is wrapped inside [ ]. Patch by Chandru. +IDLE +---- + +- Issue #6378: idle.bat now runs with the appropriate Python version rather than + the system default. Patch by Sridhar Ratnakumar. + Build ----- @@ -5046,9 +5057,6 @@ Library - Issue #6048: Now Distutils uses the tarfile module in archive_util. -- Issue #5150: IDLE's format menu now has an option to strip trailing - whitespace. - - Issue #6121: pydoc now ignores leading and trailing spaces in the argument to the 'help' function. @@ -5707,6 +5715,14 @@ Library - Windows locale mapping updated to Vista. +IDLE +---- + +- Issue #5150: IDLE's format menu now has an option to strip trailing + whitespace. + +- Issue #5847: Remove -n switch on "Edit with IDLE" menu item. + Tools/Demos ----------- @@ -5740,8 +5756,6 @@ Build - Issue #6094: Build correctly with Subversion 1.7. -- Issue #5847: Remove -n switch on "Edit with IDLE" menu item. - - Issue #5726: Make Modules/ld_so_aix return the actual exit code of the linker, rather than always exit successfully. Patch by Floris Bruynooghe. @@ -8561,9 +8575,6 @@ Library Allows the demo2 function to be executed on its own instead of only when the module is run as a script. -- Bug #813342: Start the IDLE subprocess with -Qnew if the parent is - started with that option. - - Bug #1565150: Fix subsecond processing for os.utime on Windows. - Support for MSVC 8 was added to bdist_wininst. @@ -8612,9 +8623,6 @@ Library - Bug #1531862: Do not close standard file descriptors in subprocess. -- idle: Honor the "Cancel" action in the save dialog (Debian bug - #299092). - - Fix utf-8-sig incremental decoder, which didn't recognise a BOM when the first chunk fed to the decoder started with a BOM, but was longer than 3 bytes. @@ -8857,6 +8865,15 @@ Extension Modules - The sqlite3 module was updated to pysqlite 2.4.1. +IDLE +---- + +- Bug #813342: Start the IDLE subprocess with -Qnew if the parent is + started with that option. + +- IDLE: Honor the "Cancel" action in the save dialog (Debian bug + #299092). + Tests ----- From 5182a00a324c3aa977b1ea46527e646ef0957f11 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sun, 14 Apr 2013 13:08:50 -0400 Subject: [PATCH 059/480] #17341: Include name in re error message about invalid group name. Patch by Jason Michalski. --- Lib/sre_parse.py | 6 ++++-- Lib/test/test_re.py | 11 +++++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index a0cf34414b6..75f8c9601df 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -549,7 +549,8 @@ def _parse(source, state): if not name: raise error("missing group name") if not isname(name): - raise error, "bad character in group name" + raise error("bad character in group name %r" % + name) elif sourcematch("="): # named backreference name = "" @@ -563,7 +564,8 @@ def _parse(source, state): if not name: raise error("missing group name") if not isname(name): - raise error, "bad character in group name" + raise error("bad character in backref group name " + "%r" % name) gid = state.groupdict.get(name) if gid is None: raise error, "unknown group name" diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 6ddddda9e1b..18a81a21418 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2,6 +2,7 @@ from test.test_support import precisionbigmemtest, _2G, cpython_only import re from re import Scanner +import sre_constants import sys import string import traceback @@ -886,6 +887,16 @@ def test_repeat_minmax_overflow_maxrepeat(self): self.assertRaises(OverflowError, re.compile, r".{,%d}" % MAXREPEAT) self.assertRaises(OverflowError, re.compile, r".{%d,}?" % MAXREPEAT) + def test_backref_group_name_in_exception(self): + # Issue 17341: Poor error message when compiling invalid regex + with self.assertRaisesRegexp(sre_constants.error, ''): + re.compile('(?P=)') + + def test_group_name_in_exception(self): + # Issue 17341: Poor error message when compiling invalid regex + with self.assertRaisesRegexp(sre_constants.error, '\?foo'): + re.compile('(?P)') + def run_re_tests(): from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR diff --git a/Misc/ACKS b/Misc/ACKS index 5779332ce35..78f074c76db 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -674,6 +674,7 @@ Carl Meyer Mike Meyer Piotr Meyer Steven Miale +Jason Michalski Trent Mick Tom Middleton Stan Mihai diff --git a/Misc/NEWS b/Misc/NEWS index eda841fd992..d723cb59919 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -25,6 +25,9 @@ Core and Builtins Library ------- +- Issue #17341: Include the invalid name in the error messages from re about + invalid group names. + - Issue #17016: Get rid of possible pointer wraparounds and integer overflows in the re module. Patch by Nickolai Zeldovich. From 1107c9e4d5d66213ca99719d579a19f1301a9d08 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Mon, 15 Apr 2013 09:51:54 -0400 Subject: [PATCH 060/480] Issue #17728: Specify default precision for float.format for presentation types e, f, and g. --- Doc/library/string.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index b2717a026ea..8dabbf3c369 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -453,12 +453,13 @@ The available presentation types for floating point and decimal values are: +=========+==========================================================+ | ``'e'`` | Exponent notation. Prints the number in scientific | | | notation using the letter 'e' to indicate the exponent. | + | | The default precision is ``6``. | +---------+----------------------------------------------------------+ | ``'E'`` | Exponent notation. Same as ``'e'`` except it uses an | | | upper case 'E' as the separator character. | +---------+----------------------------------------------------------+ | ``'f'`` | Fixed point. Displays the number as a fixed-point | - | | number. | + | | number. The default precision is ``6``. | +---------+----------------------------------------------------------+ | ``'F'`` | Fixed point. Same as ``'f'``. | +---------+----------------------------------------------------------+ @@ -484,7 +485,7 @@ The available presentation types for floating point and decimal values are: | | the precision. | | | | | | A precision of ``0`` is treated as equivalent to a | - | | precision of ``1``. | + | | precision of ``1``. The default precision is ``6``. | +---------+----------------------------------------------------------+ | ``'G'`` | General format. Same as ``'g'`` except switches to | | | ``'E'`` if the number gets too large. The | From 1b164ab3888bc93839e9ac7635abfb8f1a1e4a85 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Mon, 15 Apr 2013 19:08:31 +0300 Subject: [PATCH 061/480] #13510: clarify that f.readlines() is note necessary to iterate over a file. Patch by Dan Riti. --- Doc/library/io.rst | 3 +++ Doc/tutorial/inputoutput.rst | 19 ++++--------------- Misc/ACKS | 1 + 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 899408687b7..cc6cc95c86c 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -296,6 +296,9 @@ I/O Base Classes to control the number of lines read: no more lines will be read if the total size (in bytes/characters) of all lines so far exceeds *hint*. + Note that it's already possible to iterate on file objects using ``for + line in file: ...`` without calling ``file.readlines()``. + .. method:: seek(offset, whence=SEEK_SET) Change the stream position to the given byte *offset*. *offset* is diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 2f08110feef..b1611f20590 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -295,18 +295,8 @@ containing only a single newline. :: >>> f.readline() '' -``f.readlines()`` returns a list containing all the lines of data in the file. -If given an optional parameter *sizehint*, it reads that many bytes from the -file and enough more to complete a line, and returns the lines from that. This -is often used to allow efficient reading of a large file by lines, but without -having to load the entire file in memory. Only complete lines will be returned. -:: - - >>> f.readlines() - ['This is the first line of the file.\n', 'Second line of the file\n'] - -An alternative approach to reading lines is to loop over the file object. This is -memory efficient, fast, and leads to simpler code:: +For reading lines from a file, you can loop over the file object. This is memory +efficient, fast, and leads to simple code:: >>> for line in f: print line, @@ -314,9 +304,8 @@ memory efficient, fast, and leads to simpler code:: This is the first line of the file. Second line of the file -The alternative approach is simpler but does not provide as fine-grained -control. Since the two approaches manage line buffering differently, they -should not be mixed. +If you want to read all the lines of a file in a list you can also use +``list(f)`` or ``f.readlines()``. ``f.write(string)`` writes the contents of *string* to the file, returning ``None``. :: diff --git a/Misc/ACKS b/Misc/ACKS index 78f074c76db..e9ec9c87fc2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -838,6 +838,7 @@ Nicholas Riley Jean-Claude Rimbault Vlad Riscutia Wes Rishel +Dan Riti Juan M. Bello Rivas Davide Rizzo Anthony Roach From 25930f977e73d13863a2b49b0be6f526d8135790 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 15 Apr 2013 21:20:14 +0200 Subject: [PATCH 062/480] Issue #17703: Fix a regression where an illegal use of Py_DECREF() after interpreter finalization can cause a crash. --- Include/object.h | 16 +++++++++++----- Misc/NEWS | 3 +++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Include/object.h b/Include/object.h index 1ba33eb6c1c..afbc68dc04e 100644 --- a/Include/object.h +++ b/Include/object.h @@ -984,16 +984,22 @@ PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(void); #define PyTrash_UNWIND_LEVEL 50 +/* Note the workaround for when the thread state is NULL (issue #17703) */ #define Py_TRASHCAN_SAFE_BEGIN(op) \ do { \ PyThreadState *_tstate = PyThreadState_GET(); \ - if (_tstate->trash_delete_nesting < PyTrash_UNWIND_LEVEL) { \ - ++_tstate->trash_delete_nesting; + if (!_tstate || \ + _tstate->trash_delete_nesting < PyTrash_UNWIND_LEVEL) { \ + if (_tstate) \ + ++_tstate->trash_delete_nesting; /* The body of the deallocator is here. */ #define Py_TRASHCAN_SAFE_END(op) \ - --_tstate->trash_delete_nesting; \ - if (_tstate->trash_delete_later && _tstate->trash_delete_nesting <= 0) \ - _PyTrash_thread_destroy_chain(); \ + if (_tstate) { \ + --_tstate->trash_delete_nesting; \ + if (_tstate->trash_delete_later \ + && _tstate->trash_delete_nesting <= 0) \ + _PyTrash_thread_destroy_chain(); \ + } \ } \ else \ _PyTrash_thread_deposit_object((PyObject*)op); \ diff --git a/Misc/NEWS b/Misc/NEWS index d723cb59919..3ccdb10bfa5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,9 @@ Build Core and Builtins ----------------- +- Issue #17703: Fix a regression where an illegal use of Py_DECREF() after + interpreter finalization can cause a crash. + - Issue #16447: Fixed potential segmentation fault when setting __name__ on a class. From 4a67a7e53543224562ef5716f4cd8a93941dd9ea Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 15 Apr 2013 21:35:25 +0200 Subject: [PATCH 063/480] Issue #17710: Fix cPickle raising a SystemError on bogus input. --- Lib/pickle.py | 2 +- Lib/test/pickletester.py | 2 ++ Misc/NEWS | 2 ++ Modules/cPickle.c | 10 ++++++---- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Lib/pickle.py b/Lib/pickle.py index 5b95cbaca76..508e858d816 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -962,7 +962,7 @@ def load_string(self): rep = self.readline()[:-1] for q in "\"'": # double or single quote if rep.startswith(q): - if not rep.endswith(q): + if len(rep) < 2 or not rep.endswith(q): raise ValueError, "insecure string pickle" rep = rep[len(q):-len(q)] break diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 7f43dfb90ba..34cafcb7b86 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -538,6 +538,8 @@ def test_insecure_strings(self): "'abc\"", # open quote and close quote don't match "'abc' ?", # junk after close quote "'\\'", # trailing backslash + "'", # issue #17710 + "' ", # issue #17710 # some tests of the quoting rules #"'abc\"\''", #"'\\\\a\'\'\'\\\'\\\\\''", diff --git a/Misc/NEWS b/Misc/NEWS index 3ccdb10bfa5..fc817401dfe 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,6 +28,8 @@ Core and Builtins Library ------- +- Issue #17710: Fix cPickle raising a SystemError on bogus input. + - Issue #17341: Include the invalid name in the error messages from re about invalid group names. diff --git a/Modules/cPickle.c b/Modules/cPickle.c index d74ec5b7bba..8145bbf381a 100644 --- a/Modules/cPickle.c +++ b/Modules/cPickle.c @@ -3643,17 +3643,19 @@ load_string(Unpicklerobject *self) /* Strip outermost quotes */ - while (s[len-1] <= ' ') + while (len > 0 && s[len-1] <= ' ') len--; - if(s[0]=='"' && s[len-1]=='"'){ + if (len > 1 && s[0]=='"' && s[len-1]=='"') { s[len-1] = '\0'; p = s + 1 ; len -= 2; - } else if(s[0]=='\'' && s[len-1]=='\''){ + } + else if (len > 1 && s[0]=='\'' && s[len-1]=='\'') { s[len-1] = '\0'; p = s + 1 ; len -= 2; - } else + } + else goto insecure; /********************************************/ From 51bad0eb8f650e9fd5df55183538cd88fa86e709 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 16 Apr 2013 20:27:17 +0200 Subject: [PATCH 064/480] Issue #17739: fix the description of SSLSocket.getpeercert(binary_form=True) for server sockets. Thanks to David D Lowe for reporting. --- Doc/library/ssl.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 1ddc0f7b423..01072c458bb 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -328,7 +328,7 @@ SSL sockets also have the following additional methods and attributes: If there is no certificate for the peer on the other end of the connection, returns ``None``. - If the parameter ``binary_form`` is :const:`False`, and a certificate was + If the ``binary_form`` parameter is :const:`False`, and a certificate was received from the peer, this method returns a :class:`dict` instance. If the certificate was not validated, the dict is empty. If the certificate was validated, it returns a dict with the keys ``subject`` (the principal for @@ -354,10 +354,16 @@ SSL sockets also have the following additional methods and attributes: If the ``binary_form`` parameter is :const:`True`, and a certificate was provided, this method returns the DER-encoded form of the entire certificate as a sequence of bytes, or :const:`None` if the peer did not provide a - certificate. This return value is independent of validation; if validation - was required (:const:`CERT_OPTIONAL` or :const:`CERT_REQUIRED`), it will have - been validated, but if :const:`CERT_NONE` was used to establish the - connection, the certificate, if present, will not have been validated. + certificate. Whether the peer provides a certificate depends on the SSL + socket's role: + + * for a client SSL socket, the server will always provide a certificate, + regardless of whether validation was required; + + * for a server SSL socket, the client will only provide a certificate + when requested by the server; therefore :meth:`getpeercert` will return + :const:`None` if you used :const:`CERT_NONE` (rather than + :const:`CERT_OPTIONAL` or :const:`CERT_REQUIRED`). .. method:: SSLSocket.cipher() From 54167f86c75fad0c971c295f4e0fc0ef31c702fe Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Wed, 17 Apr 2013 04:10:26 +0300 Subject: [PATCH 065/480] #17740: fix links to the socket function. Initial patch by Zachary Ware. --- Doc/library/socket.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index d6ac36431fb..453c9447441 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -28,7 +28,7 @@ want to refer to :rfc:`3493` titled Basic Socket Interface Extensions for IPv6. The Python interface is a straightforward transliteration of the Unix system call and library interface for sockets to Python's object-oriented style: the -:func:`socket` function returns a :dfn:`socket object` whose methods implement +:func:`.socket` function returns a :dfn:`socket object` whose methods implement the various socket system calls. Parameter types are somewhat higher-level than in the C interface: as with :meth:`read` and :meth:`write` operations on Python files, buffer allocation on receive operations is automatic, and buffer length @@ -146,7 +146,7 @@ The module :mod:`socket` exports the following constants and functions: AF_INET6 These constants represent the address (and protocol) families, used for the - first argument to :func:`socket`. If the :const:`AF_UNIX` constant is not + first argument to :func:`.socket`. If the :const:`AF_UNIX` constant is not defined then this protocol is unsupported. @@ -252,7 +252,7 @@ The module :mod:`socket` exports the following constants and functions: ``(family, socktype, proto, canonname, sockaddr)`` In these tuples, *family*, *socktype*, *proto* are all integers and are - meant to be passed to the :func:`socket` function. *canonname* will be + meant to be passed to the :func:`.socket` function. *canonname* will be a string representing the canonical name of the *host* if :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* will be empty. *sockaddr* is a tuple describing a socket address, whose @@ -343,7 +343,7 @@ The module :mod:`socket` exports the following constants and functions: .. function:: getprotobyname(protocolname) Translate an Internet protocol name (for example, ``'icmp'``) to a constant - suitable for passing as the (optional) third argument to the :func:`socket` + suitable for passing as the (optional) third argument to the :func:`.socket` function. This is usually only needed for sockets opened in "raw" mode (:const:`SOCK_RAW`); for the normal socket modes, the correct protocol is chosen automatically if the protocol is omitted or zero. @@ -377,7 +377,7 @@ The module :mod:`socket` exports the following constants and functions: Build a pair of connected socket objects using the given address family, socket type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`socket` function above. The default family is :const:`AF_UNIX` + as for the :func:`.socket` function above. The default family is :const:`AF_UNIX` if defined on the platform; otherwise, the default is :const:`AF_INET`. Availability: Unix. @@ -388,7 +388,7 @@ The module :mod:`socket` exports the following constants and functions: Duplicate the file descriptor *fd* (an integer as returned by a file object's :meth:`fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`socket` function + family, socket type and protocol number are as for the :func:`.socket` function above. The file descriptor should refer to a socket, but this is not checked --- subsequent operations on the object may fail if the file descriptor is invalid. This function is rarely needed, but can be used to get or set socket options on @@ -861,10 +861,10 @@ Example Here are four minimal example programs using the TCP/IP protocol: a server that echoes all data that it receives back (servicing only one client), and a client -using it. Note that a server must perform the sequence :func:`socket`, +using it. Note that a server must perform the sequence :func:`.socket`, :meth:`~socket.bind`, :meth:`~socket.listen`, :meth:`~socket.accept` (possibly repeating the :meth:`~socket.accept` to service more than one client), while a -client only needs the sequence :func:`socket`, :meth:`~socket.connect`. Also +client only needs the sequence :func:`.socket`, :meth:`~socket.connect`. Also note that the server does not :meth:`~socket.sendall`/:meth:`~socket.recv` on the socket it is listening on but on the new socket returned by :meth:`~socket.accept`. From 9cfa53acfc0b10228b2f321c1c422c749e8a4f5f Mon Sep 17 00:00:00 2001 From: Roger Serwy Date: Wed, 17 Apr 2013 00:22:50 -0500 Subject: [PATCH 066/480] #14735: Update IDLE docs to omit "Control-z on Windows". --- Lib/idlelib/help.txt | 3 +-- Misc/NEWS | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/help.txt b/Lib/idlelib/help.txt index 2ebf8bf02a5..4b42e0536cc 100644 --- a/Lib/idlelib/help.txt +++ b/Lib/idlelib/help.txt @@ -233,8 +233,7 @@ Completions: Python Shell window: Control-c interrupts executing command. - Control-d sends end-of-file; closes window if typed at >>> prompt - (this is Control-z on Windows). + Control-d sends end-of-file; closes window if typed at >>> prompt. Command history: diff --git a/Misc/NEWS b/Misc/NEWS index fc817401dfe..5b4080ac6ab 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -56,6 +56,8 @@ Library IDLE ---- +- Issue #14735: Update IDLE docs to omit "Control-z on Windows". + - Issue #17585: Fixed IDLE regression. Now closes when using exit() or quit(). - Issue #17657: Show full Tk version in IDLE's about dialog. From 8ee370f44a7afb8d4c3ab82d1e824fe606512f1c Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Wed, 17 Apr 2013 19:15:52 +0100 Subject: [PATCH 067/480] Issue #17555: Fix ForkAwareThreadLock so that size of after fork registry does not grow exponentially with generation of process. --- Lib/multiprocessing/util.py | 5 ++++- Lib/test/test_multiprocessing.py | 32 +++++++++++++++++++++++++++++++- Misc/NEWS | 3 +++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 3ce480fc420..d1b3d2e3e87 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -329,10 +329,13 @@ def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers, class ForkAwareThreadLock(object): def __init__(self): + self._reset() + register_after_fork(self, ForkAwareThreadLock._reset) + + def _reset(self): self._lock = threading.Lock() self.acquire = self._lock.acquire self.release = self._lock.release - register_after_fork(self, ForkAwareThreadLock.__init__) class ForkAwareLocal(threading.local): def __init__(self): diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index bdf4e622490..88663ef55bb 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -2430,13 +2430,43 @@ def test_flags(self): [sys.executable, '-E', '-B', '-O', '-c', prog]) child_flags, grandchild_flags = json.loads(data.decode('ascii')) self.assertEqual(child_flags, grandchild_flags) + +# +# Issue #17555: ForkAwareThreadLock +# + +class TestForkAwareThreadLock(unittest.TestCase): + # We recurisvely start processes. Issue #17555 meant that the + # after fork registry would get duplicate entries for the same + # lock. The size of the registry at generation n was ~2**n. + + @classmethod + def child(cls, n, conn): + if n > 1: + p = multiprocessing.Process(target=cls.child, args=(n-1, conn)) + p.start() + p.join() + else: + conn.send(len(util._afterfork_registry)) + conn.close() + + def test_lock(self): + r, w = multiprocessing.Pipe(False) + l = util.ForkAwareThreadLock() + old_size = len(util._afterfork_registry) + p = multiprocessing.Process(target=self.child, args=(5, w)) + p.start() + new_size = r.recv() + p.join() + self.assertLessEqual(new_size, old_size) + # # # testcases_other = [OtherTest, TestInvalidHandle, TestInitializers, TestStdinBadfiledescriptor, TestTimeouts, TestNoForkBomb, - TestFlags] + TestFlags, TestForkAwareThreadLock] # # diff --git a/Misc/NEWS b/Misc/NEWS index 5b4080ac6ab..9c651d31417 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,6 +28,9 @@ Core and Builtins Library ------- +- Issue #17555: Fix ForkAwareThreadLock so that size of after fork + registry does not grow exponentially with generation of process. + - Issue #17710: Fix cPickle raising a SystemError on bogus input. - Issue #17341: Include the invalid name in the error messages from re about From 384f1ec89431753a89542484c7c27df4dadbafb5 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 20 Apr 2013 13:37:50 -0400 Subject: [PATCH 068/480] #17409: Document RLIM_INFINITY and use it to clarify the setrlimit docs. --- Doc/library/resource.rst | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 834daceb8cc..7ca45347f11 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -42,6 +42,11 @@ which cannot be checked or controlled by the operating system are not defined in this module for those platforms. +.. data:: RLIM_INFINITY + + Constant used to represent the the limit for an unlimited resource. + + .. function:: getrlimit(resource) Returns a tuple ``(soft, hard)`` with the current soft and hard limits of @@ -53,12 +58,20 @@ this module for those platforms. Sets new limits of consumption of *resource*. The *limits* argument must be a tuple ``(soft, hard)`` of two integers describing the new limits. A value of - ``-1`` can be used to specify the maximum possible upper limit. + :data:`~resource.RLIM_INFINITY` can be used to request a limit that is + unlimited. Raises :exc:`ValueError` if an invalid resource is specified, if the new soft - limit exceeds the hard limit, or if a process tries to raise its hard limit - (unless the process has an effective UID of super-user). Can also raise - :exc:`error` if the underlying system call fails. + limit exceeds the hard limit, or if a process tries to raise its hard limit. + Specifying a limit of :data:`~resource.RLIM_INFINITY` when the hard or + system limit for that resource is not unlimited will result in a + :exc:`ValueError`. A process with the effective UID of super-user can + request any valid limit value, including unlimited, but :exc:`ValueError` + will still be raised if the requested limit exceeds the system imposed + limit. + + ``setrlimit`` may also raise :exc:`error` if the underlying system call + fails. These symbols define resources whose consumption can be controlled using the :func:`setrlimit` and :func:`getrlimit` functions described below. The values of From c849a97f25c07349828407d9fcd72a9455f4b8db Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 20 Apr 2013 14:07:05 -0700 Subject: [PATCH 069/480] Issue #17801: fix shebang line of gprof2html.py --- Tools/scripts/gprof2html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/gprof2html.py b/Tools/scripts/gprof2html.py index cb01c2ce680..76408ca4d6b 100755 --- a/Tools/scripts/gprof2html.py +++ b/Tools/scripts/gprof2html.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python2.3 +#! /usr/bin/env python """Transform gprof(1) output into useful HTML.""" From de70e3ac42c134d69953302f63081e834506aaf4 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sun, 21 Apr 2013 10:13:43 -0400 Subject: [PATCH 070/480] #17065: Use process-unique key for winreg test. Patch by Jeremy Kloth. --- Lib/test/test_winreg.py | 7 +++++-- Misc/NEWS | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 2799e194802..683938acd45 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -28,9 +28,12 @@ # tests are only valid up until 6.1 HAS_REFLECTION = True if WIN_VER < (6, 1) else False -test_key_name = "SOFTWARE\\Python Registry Test Key - Delete Me" +# Use a per-process key to prevent concurrent test runs (buildbot!) from +# stomping on each other. +test_key_base = "Python Test Key [%d] - Delete Me" % (os.getpid(),) +test_key_name = "SOFTWARE\\" + test_key_base # On OS'es that support reflection we should test with a reflected key -test_reflect_key_name = "SOFTWARE\\Classes\\Python Test Key - Delete Me" +test_reflect_key_name = "SOFTWARE\\Classes\\" + test_key_base test_data = [ ("Int Value", 45, REG_DWORD), diff --git a/Misc/NEWS b/Misc/NEWS index 9c651d31417..89456d7dfc2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -56,6 +56,12 @@ Library - Issue #17526: fix an IndexError raised while passing code without filename to inspect.findsource(). Initial patch by Tyler Doyle. +Tests +----- + +- Issue #17065: Use process-unique key for winreg tests to avoid failures if + test is run multiple times in parallel (eg: on a buildbot host). + IDLE ---- From b3db57db71e2e16813166cbbd48d51bcb5cff4fe Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sun, 21 Apr 2013 13:04:10 -0700 Subject: [PATCH 071/480] Issue #17670: Provide an example of expandtabs() usage. --- Doc/library/stdtypes.rst | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 1292e9bfd41..d4bf4484825 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -931,10 +931,22 @@ string functions based on regular expressions. .. method:: str.expandtabs([tabsize]) Return a copy of the string where all tab characters are replaced by one or - more spaces, depending on the current column and the given tab size. The - column number is reset to zero after each newline occurring in the string. - If *tabsize* is not given, a tab size of ``8`` characters is assumed. This - doesn't understand other non-printing characters or escape sequences. + more spaces, depending on the current column and the given tab size. Tab + positions occur every *tabsize* characters (default is 8, giving tab + positions at columns 0, 8, 16 and so on). To expand the string, the current + column is set to zero and the string is examined character by character. If + the character is a tab (``\t``), one or more space characters are inserted + in the result until the current column is equal to the next tab position. + (The tab character itself is not copied.) If the character is a newline + (``\n``) or return (``\r``), it is copied and the current column is reset to + zero. Any other character is copied unchanged and the current column is + incremented by one regardless of how the character is represented when + printed. + + >>> '01\t012\t0123\t01234'.expandtabs() + '01 012 0123 01234' + >>> '01\t012\t0123\t01234'.expandtabs(4) + '01 012 0123 01234' .. method:: str.find(sub[, start[, end]]) From c3efea37e68545814a1d9ffcbb4e630ac3a25402 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sun, 21 Apr 2013 17:14:40 -0400 Subject: [PATCH 072/480] #15575: Clarify tutorial description of when modules are executed. --- Doc/tutorial/modules.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index f5d285ec312..bcbb761b785 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -71,7 +71,8 @@ More on Modules A module can contain executable statements as well as function definitions. These statements are intended to initialize the module. They are executed only -the *first* time the module is imported somewhere. [#]_ +the *first* time the module name is encountered in an import statement. [#]_ +(They are also run if the file is executed as a script.) Each module has its own private symbol table, which is used as the global symbol table by all functions defined in the module. Thus, the author of a module can From edfd216a6d16ff5476de1fb13a6e763b45ac522c Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Mon, 22 Apr 2013 09:58:51 +0100 Subject: [PATCH 073/480] Issue #17795: Reverted backwards-incompatible change in SysLogHandler with Unix domain sockets. --- Lib/logging/handlers.py | 35 +++++++++++++++++++++++++++++------ Misc/NEWS | 3 +++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 26dfe4a1d18..5837f4eda35 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1,4 +1,4 @@ -# Copyright 2001-2012 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2013 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -18,7 +18,7 @@ Additional handlers for the logging package for Python. The core package is based on PEP 282 and comments thereto in comp.lang.python. -Copyright (C) 2001-2012 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2013 Vinay Sajip. All Rights Reserved. To use, simply 'import logging.handlers' and log away! """ @@ -737,13 +737,17 @@ class SysLogHandler(logging.Handler): } def __init__(self, address=('localhost', SYSLOG_UDP_PORT), - facility=LOG_USER, socktype=socket.SOCK_DGRAM): + facility=LOG_USER, socktype=None): """ Initialize a handler. If address is specified as a string, a UNIX socket is used. To log to a local syslogd, "SysLogHandler(address="/dev/log")" can be used. - If facility is not specified, LOG_USER is used. + If facility is not specified, LOG_USER is used. If socktype is + specified as socket.SOCK_DGRAM or socket.SOCK_STREAM, that specific + socket type will be used. For Unix sockets, you can also specify a + socktype of None, in which case socket.SOCK_DGRAM will be used, falling + back to socket.SOCK_STREAM. """ logging.Handler.__init__(self) @@ -756,18 +760,37 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), self._connect_unixsocket(address) else: self.unixsocket = 0 + if socktype is None: + socktype = socket.SOCK_DGRAM self.socket = socket.socket(socket.AF_INET, socktype) if socktype == socket.SOCK_STREAM: self.socket.connect(address) + self.socktype = socktype self.formatter = None def _connect_unixsocket(self, address): - self.socket = socket.socket(socket.AF_UNIX, self.socktype) + use_socktype = self.socktype + if use_socktype is None: + use_socktype = socket.SOCK_DGRAM + self.socket = socket.socket(socket.AF_UNIX, use_socktype) try: self.socket.connect(address) + # it worked, so set self.socktype to the used type + self.socktype = use_socktype except socket.error: self.socket.close() - raise + if self.socktype is not None: + # user didn't specify falling back, so fail + raise + use_socktype = socket.SOCK_STREAM + self.socket = socket.socket(socket.AF_UNIX, use_socktype) + try: + self.socket.connect(address) + # it worked, so set self.socktype to the used type + self.socktype = use_socktype + except socket.error: + self.socket.close() + raise # curious: when talking to the unix-domain '/dev/log' socket, a # zero-terminator seems to be required. this string is placed diff --git a/Misc/NEWS b/Misc/NEWS index 89456d7dfc2..d633536b2cb 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,6 +28,9 @@ Core and Builtins Library ------- +- Issue #17795: Reverted backwards-incompatible change in SysLogHandler with + Unix domain sockets. + - Issue #17555: Fix ForkAwareThreadLock so that size of after fork registry does not grow exponentially with generation of process. From 2d44aad7ebcb313ca936c0bf2e13effe906beece Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 22 Apr 2013 22:51:00 +0300 Subject: [PATCH 074/480] Issue #11714: Use 'with' statements to assure a Semaphore releases a condition variable. Original patch by Thomas Rachel. --- Lib/threading.py | 42 ++++++++++++++++++++---------------------- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/Lib/threading.py b/Lib/threading.py index cb49c4a7ff8..225448bf561 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -457,21 +457,20 @@ def acquire(self, blocking=1): """ rc = False - self.__cond.acquire() - while self.__value == 0: - if not blocking: - break - if __debug__: - self._note("%s.acquire(%s): blocked waiting, value=%s", - self, blocking, self.__value) - self.__cond.wait() - else: - self.__value = self.__value - 1 - if __debug__: - self._note("%s.acquire: success, value=%s", - self, self.__value) - rc = True - self.__cond.release() + with self.__cond: + while self.__value == 0: + if not blocking: + break + if __debug__: + self._note("%s.acquire(%s): blocked waiting, value=%s", + self, blocking, self.__value) + self.__cond.wait() + else: + self.__value = self.__value - 1 + if __debug__: + self._note("%s.acquire: success, value=%s", + self, self.__value) + rc = True return rc __enter__ = acquire @@ -483,13 +482,12 @@ def release(self): to become larger than zero again, wake up that thread. """ - self.__cond.acquire() - self.__value = self.__value + 1 - if __debug__: - self._note("%s.release: success, value=%s", - self, self.__value) - self.__cond.notify() - self.__cond.release() + with self.__cond: + self.__value = self.__value + 1 + if __debug__: + self._note("%s.release: success, value=%s", + self, self.__value) + self.__cond.notify() def __exit__(self, t, v, tb): self.release() diff --git a/Misc/ACKS b/Misc/ACKS index e9ec9c87fc2..8a316a395b5 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -810,6 +810,7 @@ Fernando Pérez Eduardo Pérez Brian Quinlan Anders Qvist +Thomas Rachel Burton Radons Jeff Ramnani Brodie Rao diff --git a/Misc/NEWS b/Misc/NEWS index d633536b2cb..8b0bce46b1b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,6 +28,9 @@ Core and Builtins Library ------- +- Issue #11714: Use 'with' statements to assure a Semaphore releases a + condition variable. Original patch by Thomas Rachel. + - Issue #17795: Reverted backwards-incompatible change in SysLogHandler with Unix domain sockets. From 4641e11b4ce57b872240f5ed024f020663169140 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Tue, 23 Apr 2013 09:24:51 +0300 Subject: [PATCH 075/480] #17729: remove the outdated Advocacy HOWTO, as suggested by the author. --- Doc/howto/advocacy.rst | 355 ----------------------------------------- 1 file changed, 355 deletions(-) delete mode 100644 Doc/howto/advocacy.rst diff --git a/Doc/howto/advocacy.rst b/Doc/howto/advocacy.rst deleted file mode 100644 index 2969d266adf..00000000000 --- a/Doc/howto/advocacy.rst +++ /dev/null @@ -1,355 +0,0 @@ -************************* - Python Advocacy HOWTO -************************* - -:Author: A.M. Kuchling -:Release: 0.03 - - -.. topic:: Abstract - - It's usually difficult to get your management to accept open source software, - and Python is no exception to this rule. This document discusses reasons to use - Python, strategies for winning acceptance, facts and arguments you can use, and - cases where you *shouldn't* try to use Python. - - -Reasons to Use Python -===================== - -There are several reasons to incorporate a scripting language into your -development process, and this section will discuss them, and why Python has some -properties that make it a particularly good choice. - - -Programmability ---------------- - -Programs are often organized in a modular fashion. Lower-level operations are -grouped together, and called by higher-level functions, which may in turn be -used as basic operations by still further upper levels. - -For example, the lowest level might define a very low-level set of functions for -accessing a hash table. The next level might use hash tables to store the -headers of a mail message, mapping a header name like ``Date`` to a value such -as ``Tue, 13 May 1997 20:00:54 -0400``. A yet higher level may operate on -message objects, without knowing or caring that message headers are stored in a -hash table, and so forth. - -Often, the lowest levels do very simple things; they implement a data structure -such as a binary tree or hash table, or they perform some simple computation, -such as converting a date string to a number. The higher levels then contain -logic connecting these primitive operations. Using the approach, the primitives -can be seen as basic building blocks which are then glued together to produce -the complete product. - -Why is this design approach relevant to Python? Because Python is well suited -to functioning as such a glue language. A common approach is to write a Python -module that implements the lower level operations; for the sake of speed, the -implementation might be in C, Java, or even Fortran. Once the primitives are -available to Python programs, the logic underlying higher level operations is -written in the form of Python code. The high-level logic is then more -understandable, and easier to modify. - -John Ousterhout wrote a paper that explains this idea at greater length, -entitled "Scripting: Higher Level Programming for the 21st Century". I -recommend that you read this paper; see the references for the URL. Ousterhout -is the inventor of the Tcl language, and therefore argues that Tcl should be -used for this purpose; he only briefly refers to other languages such as Python, -Perl, and Lisp/Scheme, but in reality, Ousterhout's argument applies to -scripting languages in general, since you could equally write extensions for any -of the languages mentioned above. - - -Prototyping ------------ - -In *The Mythical Man-Month*, Fredrick Brooks suggests the following rule when -planning software projects: "Plan to throw one away; you will anyway." Brooks -is saying that the first attempt at a software design often turns out to be -wrong; unless the problem is very simple or you're an extremely good designer, -you'll find that new requirements and features become apparent once development -has actually started. If these new requirements can't be cleanly incorporated -into the program's structure, you're presented with two unpleasant choices: -hammer the new features into the program somehow, or scrap everything and write -a new version of the program, taking the new features into account from the -beginning. - -Python provides you with a good environment for quickly developing an initial -prototype. That lets you get the overall program structure and logic right, and -you can fine-tune small details in the fast development cycle that Python -provides. Once you're satisfied with the GUI interface or program output, you -can translate the Python code into C++, Fortran, Java, or some other compiled -language. - -Prototyping means you have to be careful not to use too many Python features -that are hard to implement in your other language. Using ``eval()``, or regular -expressions, or the :mod:`pickle` module, means that you're going to need C or -Java libraries for formula evaluation, regular expressions, and serialization, -for example. But it's not hard to avoid such tricky code, and in the end the -translation usually isn't very difficult. The resulting code can be rapidly -debugged, because any serious logical errors will have been removed from the -prototype, leaving only more minor slip-ups in the translation to track down. - -This strategy builds on the earlier discussion of programmability. Using Python -as glue to connect lower-level components has obvious relevance for constructing -prototype systems. In this way Python can help you with development, even if -end users never come in contact with Python code at all. If the performance of -the Python version is adequate and corporate politics allow it, you may not need -to do a translation into C or Java, but it can still be faster to develop a -prototype and then translate it, instead of attempting to produce the final -version immediately. - -One example of this development strategy is Microsoft Merchant Server. Version -1.0 was written in pure Python, by a company that subsequently was purchased by -Microsoft. Version 2.0 began to translate the code into C++, shipping with some -C++code and some Python code. Version 3.0 didn't contain any Python at all; all -the code had been translated into C++. Even though the product doesn't contain -a Python interpreter, the Python language has still served a useful purpose by -speeding up development. - -This is a very common use for Python. Past conference papers have also -described this approach for developing high-level numerical algorithms; see -David M. Beazley and Peter S. Lomdahl's paper "Feeding a Large-scale Physics -Application to Python" in the references for a good example. If an algorithm's -basic operations are things like "Take the inverse of this 4000x4000 matrix", -and are implemented in some lower-level language, then Python has almost no -additional performance cost; the extra time required for Python to evaluate an -expression like ``m.invert()`` is dwarfed by the cost of the actual computation. -It's particularly good for applications where seemingly endless tweaking is -required to get things right. GUI interfaces and Web sites are prime examples. - -The Python code is also shorter and faster to write (once you're familiar with -Python), so it's easier to throw it away if you decide your approach was wrong; -if you'd spent two weeks working on it instead of just two hours, you might -waste time trying to patch up what you've got out of a natural reluctance to -admit that those two weeks were wasted. Truthfully, those two weeks haven't -been wasted, since you've learnt something about the problem and the technology -you're using to solve it, but it's human nature to view this as a failure of -some sort. - - -Simplicity and Ease of Understanding ------------------------------------- - -Python is definitely *not* a toy language that's only usable for small tasks. -The language features are general and powerful enough to enable it to be used -for many different purposes. It's useful at the small end, for 10- or 20-line -scripts, but it also scales up to larger systems that contain thousands of lines -of code. - -However, this expressiveness doesn't come at the cost of an obscure or tricky -syntax. While Python has some dark corners that can lead to obscure code, there -are relatively few such corners, and proper design can isolate their use to only -a few classes or modules. It's certainly possible to write confusing code by -using too many features with too little concern for clarity, but most Python -code can look a lot like a slightly-formalized version of human-understandable -pseudocode. - -In *The New Hacker's Dictionary*, Eric S. Raymond gives the following definition -for "compact": - -.. epigraph:: - - Compact *adj.* Of a design, describes the valuable property that it can all be - apprehended at once in one's head. This generally means the thing created from - the design can be used with greater facility and fewer errors than an equivalent - tool that is not compact. Compactness does not imply triviality or lack of - power; for example, C is compact and FORTRAN is not, but C is more powerful than - FORTRAN. Designs become non-compact through accreting features and cruft that - don't merge cleanly into the overall design scheme (thus, some fans of Classic C - maintain that ANSI C is no longer compact). - - (From http://www.catb.org/~esr/jargon/html/C/compact.html) - -In this sense of the word, Python is quite compact, because the language has -just a few ideas, which are used in lots of places. Take namespaces, for -example. Import a module with ``import math``, and you create a new namespace -called ``math``. Classes are also namespaces that share many of the properties -of modules, and have a few of their own; for example, you can create instances -of a class. Instances? They're yet another namespace. Namespaces are currently -implemented as Python dictionaries, so they have the same methods as the -standard dictionary data type: .keys() returns all the keys, and so forth. - -This simplicity arises from Python's development history. The language syntax -derives from different sources; ABC, a relatively obscure teaching language, is -one primary influence, and Modula-3 is another. (For more information about ABC -and Modula-3, consult their respective Web sites at http://www.cwi.nl/~steven/abc/ -and http://www.m3.org.) Other features have come from C, Icon, -Algol-68, and even Perl. Python hasn't really innovated very much, but instead -has tried to keep the language small and easy to learn, building on ideas that -have been tried in other languages and found useful. - -Simplicity is a virtue that should not be underestimated. It lets you learn the -language more quickly, and then rapidly write code -- code that often works the -first time you run it. - - -Java Integration ----------------- - -If you're working with Java, Jython (http://www.jython.org/) is definitely worth -your attention. Jython is a re-implementation of Python in Java that compiles -Python code into Java bytecodes. The resulting environment has very tight, -almost seamless, integration with Java. It's trivial to access Java classes -from Python, and you can write Python classes that subclass Java classes. -Jython can be used for prototyping Java applications in much the same way -CPython is used, and it can also be used for test suites for Java code, or -embedded in a Java application to add scripting capabilities. - - -Arguments and Rebuttals -======================= - -Let's say that you've decided upon Python as the best choice for your -application. How can you convince your management, or your fellow developers, -to use Python? This section lists some common arguments against using Python, -and provides some possible rebuttals. - -**Python is freely available software that doesn't cost anything. How good can -it be?** - -Very good, indeed. These days Linux and Apache, two other pieces of open source -software, are becoming more respected as alternatives to commercial software, -but Python hasn't had all the publicity. - -Python has been around for several years, with many users and developers. -Accordingly, the interpreter has been used by many people, and has gotten most -of the bugs shaken out of it. While bugs are still discovered at intervals, -they're usually either quite obscure (they'd have to be, for no one to have run -into them before) or they involve interfaces to external libraries. The -internals of the language itself are quite stable. - -Having the source code should be viewed as making the software available for -peer review; people can examine the code, suggest (and implement) improvements, -and track down bugs. To find out more about the idea of open source code, along -with arguments and case studies supporting it, go to http://www.opensource.org. - -**Who's going to support it?** - -Python has a sizable community of developers, and the number is still growing. -The Internet community surrounding the language is an active one, and is worth -being considered another one of Python's advantages. Most questions posted to -the comp.lang.python newsgroup are quickly answered by someone. - -Should you need to dig into the source code, you'll find it's clear and -well-organized, so it's not very difficult to write extensions and track down -bugs yourself. If you'd prefer to pay for support, there are companies and -individuals who offer commercial support for Python. - -**Who uses Python for serious work?** - -Lots of people; one interesting thing about Python is the surprising diversity -of applications that it's been used for. People are using Python to: - -* Run Web sites - -* Write GUI interfaces - -* Control number-crunching code on supercomputers - -* Make a commercial application scriptable by embedding the Python interpreter - inside it - -* Process large XML data sets - -* Build test suites for C or Java code - -Whatever your application domain is, there's probably someone who's used Python -for something similar. Yet, despite being useable for such high-end -applications, Python's still simple enough to use for little jobs. - -See http://wiki.python.org/moin/OrganizationsUsingPython for a list of some of -the organizations that use Python. - -**What are the restrictions on Python's use?** - -They're practically nonexistent. Consult :ref:`history-and-license` for the full -language, but it boils down to three conditions: - -* You have to leave the copyright notice on the software; if you don't include - the source code in a product, you have to put the copyright notice in the - supporting documentation. - -* Don't claim that the institutions that have developed Python endorse your - product in any way. - -* If something goes wrong, you can't sue for damages. Practically all software - licenses contain this condition. - -Notice that you don't have to provide source code for anything that contains -Python or is built with it. Also, the Python interpreter and accompanying -documentation can be modified and redistributed in any way you like, and you -don't have to pay anyone any licensing fees at all. - -**Why should we use an obscure language like Python instead of well-known -language X?** - -I hope this HOWTO, and the documents listed in the final section, will help -convince you that Python isn't obscure, and has a healthily growing user base. -One word of advice: always present Python's positive advantages, instead of -concentrating on language X's failings. People want to know why a solution is -good, rather than why all the other solutions are bad. So instead of attacking -a competing solution on various grounds, simply show how Python's virtues can -help. - - -Useful Resources -================ - -http://www.pythonology.com/success - The Python Success Stories are a collection of stories from successful users of - Python, with the emphasis on business and corporate users. - -.. http://www.fsbassociates.com/books/pythonchpt1.htm - The first chapter of \emph{Internet Programming with Python} also - examines some of the reasons for using Python. The book is well worth - buying, but the publishers have made the first chapter available on - the Web. - -http://www.tcl.tk/doc/scripting.html - John Ousterhout's white paper on scripting is a good argument for the utility of - scripting languages, though naturally enough, he emphasizes Tcl, the language he - developed. Most of the arguments would apply to any scripting language. - -http://www.python.org/workshops/1997-10/proceedings/beazley.html - The authors, David M. Beazley and Peter S. Lomdahl, describe their use of - Python at Los Alamos National Laboratory. It's another good example of how - Python can help get real work done. This quotation from the paper has been - echoed by many people: - - .. epigraph:: - - Originally developed as a large monolithic application for massively parallel - processing systems, we have used Python to transform our application into a - flexible, highly modular, and extremely powerful system for performing - simulation, data analysis, and visualization. In addition, we describe how - Python has solved a number of important problems related to the development, - debugging, deployment, and maintenance of scientific software. - -http://pythonjournal.cognizor.com/pyj1/Everitt-Feit_interview98-V1.html - This interview with Andy Feit, discussing Infoseek's use of Python, can be used - to show that choosing Python didn't introduce any difficulties into a company's - development process, and provided some substantial benefits. - -.. http://www.python.org/psa/Commercial.html - Robin Friedrich wrote this document on how to support Python's use in - commercial projects. - -http://www.python.org/workshops/1997-10/proceedings/stein.ps - For the 6th Python conference, Greg Stein presented a paper that traced Python's - adoption and usage at a startup called eShop, and later at Microsoft. - -http://www.opensource.org - Management may be doubtful of the reliability and usefulness of software that - wasn't written commercially. This site presents arguments that show how open - source software can have considerable advantages over closed-source software. - -http://www.faqs.org/docs/Linux-mini/Advocacy.html - The Linux Advocacy mini-HOWTO was the inspiration for this document, and is also - well worth reading for general suggestions on winning acceptance for a new - technology, such as Linux or Python. In general, you won't make much progress - by simply attacking existing systems and complaining about their inadequacies; - this often ends up looking like unfocused whining. It's much better to point - out some of the many areas where Python is an improvement over other systems. - From ebe2814484966b9917236f5cf8d0cfe56a42742d Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Tue, 23 Apr 2013 09:56:30 +0300 Subject: [PATCH 076/480] #17729: remove the Advocacy HOWTO from the index. --- Doc/howto/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index 6706f894d12..e4c95b1f520 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -13,7 +13,6 @@ Currently, the HOWTOs are: .. toctree:: :maxdepth: 1 - advocacy.rst pyporting.rst cporting.rst curses.rst From bb8381763bcbd68875f049db6584e9b6f8e723bc Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Tue, 23 Apr 2013 13:47:06 +0200 Subject: [PATCH 077/480] Ensure that plistlib doesn't corrupt deeply nested datastructures Without this changeset plistlib would write empty tags for plistlib.Data objects in deeply nested datastructures. Fixes #17353 --- Lib/plistlib.py | 4 ++-- Lib/test/test_plistlib.py | 12 ++++++++++++ Misc/NEWS | 2 ++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 51944eecb0f..42897b8da8b 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -262,8 +262,8 @@ def writeValue(self, value): def writeData(self, data): self.beginElement("data") self.indentLevel -= 1 - maxlinelength = 76 - len(self.indent.replace("\t", " " * 8) * - self.indentLevel) + maxlinelength = max(16, 76 - len(self.indent.replace("\t", " " * 8) * + self.indentLevel)) for line in data.asBase64(maxlinelength).split("\n"): if line: self.writeln(line) diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index e6e0f868d3b..7859ad05723 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -135,6 +135,18 @@ def test_string(self): data2 = plistlib.writePlistToString(pl2) self.assertEqual(data, data2) + def test_indentation_array(self): + data = [[[[[[[[{'test': plistlib.Data(b'aaaaaa')}]]]]]]]] + self.assertEqual(plistlib.readPlistFromString(plistlib.writePlistToString(data)), data) + + def test_indentation_dict(self): + data = {'1': {'2': {'3': {'4': {'5': {'6': {'7': {'8': {'9': plistlib.Data(b'aaaaaa')}}}}}}}}} + self.assertEqual(plistlib.readPlistFromString(plistlib.writePlistToString(data)), data) + + def test_indentation_dict_mix(self): + data = {'1': {'2': [{'3': [[[[[{'test': plistlib.Data(b'aaaaaa')}]]]]]}]}} + self.assertEqual(plistlib.readPlistFromString(plistlib.writePlistToString(data)), data) + def test_appleformatting(self): pl = plistlib.readPlistFromString(TESTDATA) data = plistlib.writePlistToString(pl) diff --git a/Misc/NEWS b/Misc/NEWS index 8b0bce46b1b..9804b0e0fcd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,6 +28,8 @@ Core and Builtins Library ------- +- Issue #17353: Plistlib emitted empty data tags with deeply nested datastructures + - Issue #11714: Use 'with' statements to assure a Semaphore releases a condition variable. Original patch by Thomas Rachel. From 6b59f109617d4b3cb2b001108f9099e781ffe903 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 24 Apr 2013 23:31:38 +0200 Subject: [PATCH 078/480] Issue #17835: Fix test_io when the default OS pipe buffer size is larger than one million bytes. --- Lib/test/test_io.py | 4 ++-- Lib/test/test_support.py | 9 +++++++++ Misc/NEWS | 3 +++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 5cd28f5bea5..63a05089313 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -2880,7 +2880,7 @@ def _read(): # The buffered IO layer must check for pending signal # handlers, which in this case will invoke alarm_interrupt(). self.assertRaises(ZeroDivisionError, - wio.write, item * (3 * 1000 * 1000)) + wio.write, item * (support.PIPE_MAX_SIZE // len(item) + 1)) t.join() # We got one byte, get another one and check that it isn't a # repeat of the first one. @@ -2978,7 +2978,7 @@ def check_interrupted_write_retry(self, item, **fdopen_kwargs): select = support.import_module("select") # A quantity that exceeds the buffer size of an anonymous pipe's # write end. - N = 1024 * 1024 + N = support.PIPE_MAX_SIZE r, w = os.pipe() fdopen_kwargs["closefd"] = False # We need a separate thread to read from the pipe and allow the diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 33de7881803..1a1cac883c6 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -400,6 +400,15 @@ def fcmp(x, y): # fuzzy comparison function return (len(x) > len(y)) - (len(x) < len(y)) return (x > y) - (x < y) + +# A constant likely larger than the underlying OS pipe buffer size, to +# make writes blocking. +# Windows limit seems to be around 512 B, and many Unix kernels have a +# 64 KiB pipe buffer size or 16 * PAGE_SIZE: take a few megs to be sure. +# (see issue #17835 for a discussion of this number). +PIPE_MAX_SIZE = 4 *1024 * 1024 + 1 + + try: unicode have_unicode = True diff --git a/Misc/NEWS b/Misc/NEWS index 9804b0e0fcd..cdcf9e5b597 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -131,6 +131,9 @@ IDLE Tests ----- +- Issue #17835: Fix test_io when the default OS pipe buffer size is larger + than one million bytes. + - Issue #17531: Fix tests that thought group and user ids were always the int type. Also, always allow -1 as a valid group and user id. From 4d3d0f211fcc199ac19570cf1ed00150331a17d5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 28 Apr 2013 14:09:47 +0300 Subject: [PATCH 079/480] Issue #17857: Prevent build failures with pre-3.5.0 versions of sqlite3, such as was shipped with Centos 5 and Mac OS X 10.4. This bug was already fixed in issue14572 for 2.7 only and then it was backported back from 3.3 in issue17073. --- Misc/NEWS | 3 +++ Modules/_sqlite/cursor.c | 2 +- Modules/_sqlite/util.c | 8 ++++---- Modules/_sqlite/util.h | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index cdcf9e5b597..ef4bc8a42b7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,9 @@ Build Core and Builtins ----------------- +- Issue #17857: Prevent build failures with pre-3.5.0 versions of sqlite3, + such as was shipped with Centos 5 and Mac OS X 10.4. + - Issue #17703: Fix a regression where an illegal use of Py_DECREF() after interpreter finalization can cause a crash. diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index ae810dea30b..f06f92c9c3b 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -732,7 +732,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* Py_DECREF(self->lastrowid); if (!multiple && statement_type == STATEMENT_INSERT) { - sqlite3_int64 lastrowid; + sqlite_int64 lastrowid; Py_BEGIN_ALLOW_THREADS lastrowid = sqlite3_last_insert_rowid(self->connection->db); Py_END_ALLOW_THREADS diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 5f06b66638b..a24dd8c6347 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -111,7 +111,7 @@ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st) #endif PyObject * -_pysqlite_long_from_int64(sqlite3_int64 value) +_pysqlite_long_from_int64(sqlite_int64 value) { #ifdef HAVE_LONG_LONG # if SIZEOF_LONG_LONG < 8 @@ -135,7 +135,7 @@ _pysqlite_long_from_int64(sqlite3_int64 value) return PyInt_FromLong(value); } -sqlite3_int64 +sqlite_int64 _pysqlite_long_as_int64(PyObject * py_val) { int overflow; @@ -158,8 +158,8 @@ _pysqlite_long_as_int64(PyObject * py_val) #endif return value; } - else if (sizeof(value) < sizeof(sqlite3_int64)) { - sqlite3_int64 int64val; + else if (sizeof(value) < sizeof(sqlite_int64)) { + sqlite_int64 int64val; if (_PyLong_AsByteArray((PyLongObject *)py_val, (unsigned char *)&int64val, sizeof(int64val), IS_LITTLE_ENDIAN, 1 /* signed */) >= 0) { diff --git a/Modules/_sqlite/util.h b/Modules/_sqlite/util.h index 12a5710ff31..4c3b2c70cb2 100644 --- a/Modules/_sqlite/util.h +++ b/Modules/_sqlite/util.h @@ -36,7 +36,7 @@ int pysqlite_step(sqlite3_stmt* statement, pysqlite_Connection* connection); */ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st); -PyObject * _pysqlite_long_from_int64(sqlite3_int64 value); -sqlite3_int64 _pysqlite_long_as_int64(PyObject * value); +PyObject * _pysqlite_long_from_int64(sqlite_int64 value); +sqlite_int64 _pysqlite_long_as_int64(PyObject * value); #endif From ad3aa97fcdd38f6767f3a32b9277021987ef41c8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 28 Apr 2013 15:48:54 +0300 Subject: [PATCH 080/480] Issue #17842. Added base64 module tests with bytearray arguments. --- Lib/test/test_base64.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index ff2c3700d3b..3f2cee4fded 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -18,6 +18,8 @@ def test_encodestring(self): "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE" "RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0\nNT" "Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==\n") + # Non-bytes + eq(base64.encodestring(bytearray('abc')), 'YWJj\n') def test_decodestring(self): eq = self.assertEqual @@ -32,6 +34,8 @@ def test_decodestring(self): "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789!@#0^&*();:<>,. []{}") eq(base64.decodestring(''), '') + # Non-bytes + eq(base64.decodestring(bytearray("YWJj\n")), "abc") def test_encode(self): eq = self.assertEqual @@ -73,6 +77,10 @@ def test_b64encode(self): "Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==") # Test with arbitrary alternative characters eq(base64.b64encode('\xd3V\xbeo\xf7\x1d', altchars='*$'), '01a*b$cd') + # Non-bytes + eq(base64.b64encode(bytearray('abcd')), 'YWJjZA==') + self.assertRaises(TypeError, base64.b64encode, + '\xd3V\xbeo\xf7\x1d', altchars=bytearray('*$')) # Test standard alphabet eq(base64.standard_b64encode("www.python.org"), "d3d3LnB5dGhvbi5vcmc=") eq(base64.standard_b64encode("a"), "YQ==") @@ -85,8 +93,12 @@ def test_b64encode(self): "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE" "RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NT" "Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==") + # Non-bytes + eq(base64.standard_b64encode(bytearray('abcd')), 'YWJjZA==') # Test with 'URL safe' alternative characters eq(base64.urlsafe_b64encode('\xd3V\xbeo\xf7\x1d'), '01a-b_cd') + # Non-bytes + eq(base64.urlsafe_b64encode(bytearray('\xd3V\xbeo\xf7\x1d')), '01a-b_cd') def test_b64decode(self): eq = self.assertEqual @@ -104,6 +116,8 @@ def test_b64decode(self): eq(base64.b64decode(''), '') # Test with arbitrary alternative characters eq(base64.b64decode('01a*b$cd', altchars='*$'), '\xd3V\xbeo\xf7\x1d') + # Non-bytes + eq(base64.b64decode(bytearray("YWJj")), "abc") # Test standard alphabet eq(base64.standard_b64decode("d3d3LnB5dGhvbi5vcmc="), "www.python.org") eq(base64.standard_b64decode("YQ=="), "a") @@ -116,8 +130,12 @@ def test_b64decode(self): "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789!@#0^&*();:<>,. []{}") + # Non-bytes + eq(base64.standard_b64decode(bytearray("YWJj")), "abc") # Test with 'URL safe' alternative characters eq(base64.urlsafe_b64decode('01a-b_cd'), '\xd3V\xbeo\xf7\x1d') + # Non-bytes + eq(base64.urlsafe_b64decode(bytearray('01a-b_cd')), '\xd3V\xbeo\xf7\x1d') def test_b64decode_error(self): self.assertRaises(TypeError, base64.b64decode, 'abc') @@ -131,6 +149,8 @@ def test_b32encode(self): eq(base64.b32encode('abc'), 'MFRGG===') eq(base64.b32encode('abcd'), 'MFRGGZA=') eq(base64.b32encode('abcde'), 'MFRGGZDF') + # Non-bytes + eq(base64.b32encode(bytearray('abcd')), 'MFRGGZA=') def test_b32decode(self): eq = self.assertEqual @@ -141,6 +161,8 @@ def test_b32decode(self): eq(base64.b32decode('MFRGG==='), 'abc') eq(base64.b32decode('MFRGGZA='), 'abcd') eq(base64.b32decode('MFRGGZDF'), 'abcde') + # Non-bytes + self.assertRaises(TypeError, base64.b32decode, bytearray('MFRGG===')) def test_b32decode_casefold(self): eq = self.assertEqual @@ -171,6 +193,8 @@ def test_b16encode(self): eq = self.assertEqual eq(base64.b16encode('\x01\x02\xab\xcd\xef'), '0102ABCDEF') eq(base64.b16encode('\x00'), '00') + # Non-bytes + eq(base64.b16encode(bytearray('\x01\x02\xab\xcd\xef')), '0102ABCDEF') def test_b16decode(self): eq = self.assertEqual @@ -180,6 +204,8 @@ def test_b16decode(self): self.assertRaises(TypeError, base64.b16decode, '0102abcdef') # Case fold eq(base64.b16decode('0102abcdef', True), '\x01\x02\xab\xcd\xef') + # Non-bytes + eq(base64.b16decode(bytearray("0102ABCDEF")), '\x01\x02\xab\xcd\xef') From 2cd051bebe7094071e526ca1a09e5e783330ccc2 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sun, 28 Apr 2013 11:16:21 -0400 Subject: [PATCH 081/480] #7152: Clarify that ProxyHandler is added only if proxy settings are detected. Behavior confirmation and initial patch by Jessica McKellar. --- Doc/howto/urllib2.rst | 8 +++++--- Doc/library/urllib2.rst | 15 +++++++++------ Misc/ACKS | 1 + 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index a8553081b57..0291c684d8c 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -489,9 +489,10 @@ than the URL you pass to .add_password() will also match. :: In the above example we only supplied our ``HTTPBasicAuthHandler`` to ``build_opener``. By default openers have the handlers for normal situations - -- ``ProxyHandler``, ``UnknownHandler``, ``HTTPHandler``, + -- ``ProxyHandler`` (if a proxy setting such as an :envvar:`http_proxy` + environment variable is set), ``UnknownHandler``, ``HTTPHandler``, ``HTTPDefaultErrorHandler``, ``HTTPRedirectHandler``, ``FTPHandler``, - ``FileHandler``, ``HTTPErrorProcessor``. + ``FileHandler``, ``DataHandler``, ``HTTPErrorProcessor``. ``top_level_url`` is in fact *either* a full URL (including the 'http:' scheme component and the hostname and optionally the port number) @@ -506,7 +507,8 @@ Proxies ======= **urllib2** will auto-detect your proxy settings and use those. This is through -the ``ProxyHandler`` which is part of the normal handler chain. Normally that's +the ``ProxyHandler``, which is part of the normal handler chain when a proxy +setting is detected. Normally that's a good thing, but there are occasions when it may not be helpful [#]_. One way to do this is to setup our own ``ProxyHandler``, with no proxies defined. This is done using similar steps to setting up a `Basic Authentication`_ handler : :: diff --git a/Doc/library/urllib2.rst b/Doc/library/urllib2.rst index bae5a4b2f5b..a35f1751c2f 100644 --- a/Doc/library/urllib2.rst +++ b/Doc/library/urllib2.rst @@ -60,8 +60,10 @@ The :mod:`urllib2` module defines the following functions: default installed global :class:`OpenerDirector` uses :class:`UnknownHandler` to ensure this never happens). - In addition, default installed :class:`ProxyHandler` makes sure the requests - are handled through the proxy when they are set. + In addition, if proxy settings are detected (for example, when a ``*_proxy`` + environment variable like :envvar:`http_proxy` is set), + :class:`ProxyHandler` is default installed and makes sure the requests are + handled through the proxy. .. versionchanged:: 2.6 *timeout* was added. @@ -83,7 +85,8 @@ The :mod:`urllib2` module defines the following functions: subclasses of :class:`BaseHandler` (in which case it must be possible to call the constructor without any parameters). Instances of the following classes will be in front of the *handler*\s, unless the *handler*\s contain them, - instances of them or subclasses of them: :class:`ProxyHandler`, + instances of them or subclasses of them: :class:`ProxyHandler` (if proxy + settings are detected), :class:`UnknownHandler`, :class:`HTTPHandler`, :class:`HTTPDefaultErrorHandler`, :class:`HTTPRedirectHandler`, :class:`FTPHandler`, :class:`FileHandler`, :class:`HTTPErrorProcessor`. @@ -202,9 +205,9 @@ The following classes are provided: Cause requests to go through a proxy. If *proxies* is given, it must be a dictionary mapping protocol names to URLs of proxies. The default is to read the list of proxies from the environment variables - :envvar:`_proxy`. If no proxy environment variables are set, in a - Windows environment, proxy settings are obtained from the registry's - Internet Settings section and in a Mac OS X environment, proxy information + :envvar:`_proxy`. If no proxy environment variables are set, then + in a Windows environment proxy settings are obtained from the registry's + Internet Settings section, and in a Mac OS X environment proxy information is retrieved from the OS X System Configuration Framework. To disable autodetected proxy pass an empty dictionary. diff --git a/Misc/ACKS b/Misc/ACKS index 8a316a395b5..9f50b4c5dca 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -656,6 +656,7 @@ Kirk McDonald Chris McDonough Greg McFarlane Alan McIntyre +Jessica McKellar Michael McLay Mark Mc Mahon Gordon McMillan From e1a41fe5b764945a36fcc8932ca7b6a661a7b450 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sun, 28 Apr 2013 11:20:46 -0400 Subject: [PATCH 082/480] Reflow paragraph. --- Doc/howto/urllib2.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 0291c684d8c..d4a0e639c01 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -508,10 +508,10 @@ Proxies **urllib2** will auto-detect your proxy settings and use those. This is through the ``ProxyHandler``, which is part of the normal handler chain when a proxy -setting is detected. Normally that's -a good thing, but there are occasions when it may not be helpful [#]_. One way -to do this is to setup our own ``ProxyHandler``, with no proxies defined. This -is done using similar steps to setting up a `Basic Authentication`_ handler : :: +setting is detected. Normally that's a good thing, but there are occasions +when it may not be helpful [#]_. One way to do this is to setup our own +``ProxyHandler``, with no proxies defined. This is done using similar steps to +setting up a `Basic Authentication`_ handler : :: >>> proxy_support = urllib2.ProxyHandler({}) >>> opener = urllib2.build_opener(proxy_support) From c2a60af6d1e8476c7a9f7db0faf9fcd7cbec1e7c Mon Sep 17 00:00:00 2001 From: Nadeem Vawda Date: Sun, 28 Apr 2013 18:43:31 +0200 Subject: [PATCH 083/480] Issue #17843: Remove bz2 test data that triggers antivirus warnings. --- Lib/test/test_bz2.py | 13 ++++++------- Lib/test/testbz2_bigmem.bz2 | Bin 3023 -> 0 bytes 2 files changed, 6 insertions(+), 7 deletions(-) delete mode 100644 Lib/test/testbz2_bigmem.bz2 diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index fdab4e2e6f8..4c49e7eb2d2 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -25,9 +25,6 @@ class BaseTest(unittest.TestCase): DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' EMPTY_DATA = 'BZh9\x17rE8P\x90\x00\x00\x00\x00' - with open(findfile("testbz2_bigmem.bz2"), "rb") as f: - DATA_BIGMEM = f.read() - if has_cmdline_bunzip2: def decompress(self, data): pop = subprocess.Popen("bunzip2", shell=True, @@ -431,9 +428,10 @@ def testBigmem(self, size): # Issue #14398: decompression fails when output data is >=2GB. if size < _4G: self.skipTest("Test needs 5GB of memory to run.") - text = bz2.BZ2Decompressor().decompress(self.DATA_BIGMEM) + compressed = bz2.compress("a" * _4G) + text = bz2.BZ2Decompressor().decompress(compressed) self.assertEqual(len(text), _4G) - self.assertEqual(text.strip("\0"), "") + self.assertEqual(text.strip("a"), "") class FuncTest(BaseTest): @@ -482,9 +480,10 @@ def testDecompressBigmem(self, size): # Issue #14398: decompression fails when output data is >=2GB. if size < _4G: self.skipTest("Test needs 5GB of memory to run.") - text = bz2.decompress(self.DATA_BIGMEM) + compressed = bz2.compress("a" * _4G) + text = bz2.decompress(compressed) self.assertEqual(len(text), _4G) - self.assertEqual(text.strip("\0"), "") + self.assertEqual(text.strip("a"), "") def test_main(): test_support.run_unittest( diff --git a/Lib/test/testbz2_bigmem.bz2 b/Lib/test/testbz2_bigmem.bz2 deleted file mode 100644 index c9a4616c8053b1c92a45023635c8610c26299fc2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3023 zcmZ>Y%CIzaj8qGb+kyeFkUYXAVC C(2;Tg From 8cc1faa44c972c408fb3f78431d94367f5a38fe2 Mon Sep 17 00:00:00 2001 From: Nadeem Vawda Date: Sun, 28 Apr 2013 19:01:31 +0200 Subject: [PATCH 084/480] Add missing NEWS entry for issue #17843. --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index ef4bc8a42b7..46c3072d7c4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -31,6 +31,9 @@ Core and Builtins Library ------- +- Issue #17843: Removed test data file that was triggering false-positive virus + warnings with certain antivirus software. + - Issue #17353: Plistlib emitted empty data tags with deeply nested datastructures - Issue #11714: Use 'with' statements to assure a Semaphore releases a From 2a4ba6d6bd860519aba57bad7238b76530682296 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sun, 28 Apr 2013 17:04:53 -0400 Subject: [PATCH 085/480] #7152: Remove incorrectly added reference to DataHandler. --- Doc/howto/urllib2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index d4a0e639c01..f84bf0810f6 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -492,7 +492,7 @@ than the URL you pass to .add_password() will also match. :: -- ``ProxyHandler`` (if a proxy setting such as an :envvar:`http_proxy` environment variable is set), ``UnknownHandler``, ``HTTPHandler``, ``HTTPDefaultErrorHandler``, ``HTTPRedirectHandler``, ``FTPHandler``, - ``FileHandler``, ``DataHandler``, ``HTTPErrorProcessor``. + ``FileHandler``, ``HTTPErrorProcessor``. ``top_level_url`` is in fact *either* a full URL (including the 'http:' scheme component and the hostname and optionally the port number) From 18cc7c8a8e4cb018f63e2a81f8479cfeaf45ba64 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Mon, 29 Apr 2013 12:26:01 +0300 Subject: [PATCH 086/480] #17866: mention that in Python 3, assertItemsEqual is named assertCountEqual. --- Doc/library/unittest.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index abdef988add..46f429c40cf 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1075,6 +1075,8 @@ Test cases sorted(actual))`` but it works with sequences of unhashable objects as well. + In Python 3, this method is named ``assertCountEqual``. + .. versionadded:: 2.7 From 771aadb6b292c1a6d72b0e6f90b2ffd6ed2afe77 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Tue, 30 Apr 2013 00:08:51 -0700 Subject: [PATCH 087/480] This local change was lost during the fixing of issue17192 to update libffi to 3.0.13. (i'm not sure if it is needed anymore but see issue 10309 for details which makes no mention of upstream) --- Modules/_ctypes/libffi/src/dlmalloc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Modules/_ctypes/libffi/src/dlmalloc.c b/Modules/_ctypes/libffi/src/dlmalloc.c index 5c9f9c2d239..2773953590e 100644 --- a/Modules/_ctypes/libffi/src/dlmalloc.c +++ b/Modules/_ctypes/libffi/src/dlmalloc.c @@ -457,6 +457,11 @@ DEFAULT_MMAP_THRESHOLD default: 256K #define LACKS_ERRNO_H #define MALLOC_FAILURE_ACTION #define MMAP_CLEARS 0 /* WINCE and some others apparently don't clear */ +#elif !defined _GNU_SOURCE +/* mremap() on Linux requires this via sys/mman.h + * See roundup issue 10309 + */ +#define _GNU_SOURCE 1 #endif /* WIN32 */ #ifdef __OS2__ From 5caf36284313dfc079d5196fc4f7d464696a1449 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Tue, 30 Apr 2013 00:54:45 -0700 Subject: [PATCH 088/480] NEWS entry --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 46c3072d7c4..6dc4c73693b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,9 @@ Build Core and Builtins ----------------- +- Issue #17192: Restore the patch for Issue #10309 which was ommitted + in 2.7.4 when updating the bundled version of libffi. + - Issue #17857: Prevent build failures with pre-3.5.0 versions of sqlite3, such as was shipped with Centos 5 and Mac OS X 10.4. From e695a3bb5ea8b461e084cfe17a7c87bb00bee357 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Tue, 30 Apr 2013 00:56:07 -0700 Subject: [PATCH 089/480] move the NEWS entry to the right section and mention ctypes. --- Misc/NEWS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 6dc4c73693b..4b7b3a263a4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,9 +17,6 @@ Build Core and Builtins ----------------- -- Issue #17192: Restore the patch for Issue #10309 which was ommitted - in 2.7.4 when updating the bundled version of libffi. - - Issue #17857: Prevent build failures with pre-3.5.0 versions of sqlite3, such as was shipped with Centos 5 and Mac OS X 10.4. @@ -34,6 +31,9 @@ Core and Builtins Library ------- +- Issue #17192: Restore the patch for Issue #10309 which was ommitted + in 2.7.4 when updating the bundled version of libffi used by ctypes. + - Issue #17843: Removed test data file that was triggering false-positive virus warnings with certain antivirus software. From 1f6722d98e125597d42746c2b4c6aceb373d0549 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Tue, 30 Apr 2013 23:33:31 +0300 Subject: [PATCH 090/480] #16518: add "bytes-like object" to the glossary. --- Doc/glossary.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 3641e5c189d..19b02f4e8f8 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -77,6 +77,10 @@ Glossary Benevolent Dictator For Life, a.k.a. `Guido van Rossum `_, Python's creator. + bytes-like object + An object that supports the :ref:`bufferobjects`, like :class:`bytes` or + :class:`bytearray`. + bytecode Python source code is compiled into bytecode, the internal representation of a Python program in the CPython interpreter. The bytecode is also From 2bbabf6444987feb0da71d641cb99e76915954dd Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 1 May 2013 00:15:44 +0200 Subject: [PATCH 091/480] Issue #17712: Fix test_gdb failures on Ubuntu 13.04. --- Lib/test/test_gdb.py | 46 +++++++++++++++++++++++--------------------- Misc/NEWS | 2 ++ 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 9d8ff8ed8a1..5302b5adb42 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -142,30 +142,32 @@ def get_stack_trace(self, source=None, script=None, # Use "args" to invoke gdb, capturing stdout, stderr: out, err = run_gdb(*args, PYTHONHASHSEED='0') - # Ignore some noise on stderr due to the pending breakpoint: - err = err.replace('Function "%s" not defined.\n' % breakpoint, '') - # Ignore some other noise on stderr (http://bugs.python.org/issue8600) - err = err.replace("warning: Unable to find libthread_db matching" - " inferior's thread library, thread debugging will" - " not be available.\n", - '') - err = err.replace("warning: Cannot initialize thread debugging" - " library: Debugger service failed\n", - '') - err = err.replace('warning: Could not load shared library symbols for ' - 'linux-vdso.so.1.\n' - 'Do you need "set solib-search-path" or ' - '"set sysroot"?\n', - '') - err = err.replace('warning: Could not load shared library symbols for ' - 'linux-gate.so.1.\n' - 'Do you need "set solib-search-path" or ' - '"set sysroot"?\n', - '') + errlines = err.splitlines() + unexpected_errlines = [] + + # Ignore some benign messages on stderr. + ignore_patterns = ( + 'Function "%s" not defined.' % breakpoint, + "warning: no loadable sections found in added symbol-file" + " system-supplied DSO", + "warning: Unable to find libthread_db matching" + " inferior's thread library, thread debugging will" + " not be available.", + "warning: Cannot initialize thread debugging" + " library: Debugger service failed", + 'warning: Could not load shared library symbols for ' + 'linux-vdso.so', + 'warning: Could not load shared library symbols for ' + 'linux-gate.so', + 'Do you need "set solib-search-path" or ' + '"set sysroot"?', + ) + for line in errlines: + if not line.startswith(ignore_patterns): + unexpected_errlines.append(line) # Ensure no unexpected error messages: - self.assertEqual(err, '') - + self.assertEqual(unexpected_errlines, []) return out def get_gdb_repr(self, source, diff --git a/Misc/NEWS b/Misc/NEWS index 4b7b3a263a4..b61bf230d18 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -76,6 +76,8 @@ Library Tests ----- +- Issue #17712: Fix test_gdb failures on Ubuntu 13.04. + - Issue #17065: Use process-unique key for winreg tests to avoid failures if test is run multiple times in parallel (eg: on a buildbot host). From 9c8efa94829ecdce10503bc66c6504c87d22f436 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Wed, 1 May 2013 14:13:05 +0300 Subject: [PATCH 092/480] #16518: fix links in glossary entry. --- Doc/glossary.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 19b02f4e8f8..0d02d1acfc8 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -78,8 +78,8 @@ Glossary `_, Python's creator. bytes-like object - An object that supports the :ref:`bufferobjects`, like :class:`bytes` or - :class:`bytearray`. + An object that supports the :ref:`buffer protocol `, + like :class:`str` or :class:`bytearray`. bytecode Python source code is compiled into bytecode, the internal representation From da1c5d951b9dd60fb8167229e2e633c84780a863 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Thu, 2 May 2013 05:50:21 -0700 Subject: [PATCH 093/480] # 1722 - Add a note on urllib helper functions like splittype, splithost etc. --- Doc/library/urllib.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/library/urllib.rst b/Doc/library/urllib.rst index 2008bdfe3e7..c7d200dbe8a 100644 --- a/Doc/library/urllib.rst +++ b/Doc/library/urllib.rst @@ -280,6 +280,13 @@ Utility functions find it, looks for proxy information from Mac OSX System Configuration for Mac OS X and Windows Systems Registry for Windows. +.. note:: + urllib also exposes certain utility functions like splittype, splithost and + others parsing url into various components. But it is recommended to use + :mod:`urlparse` for parsing urls than using these functions directly. + Python 3 does not expose these helper functions from :mod:`urllib.parse` + module. + URL Opener objects ------------------ From f76d22f861b9498f852817532ccdda4df7bcc359 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 3 May 2013 00:59:20 -0700 Subject: [PATCH 094/480] Issue #15535: Fix regression in pickling of named tuples. --- Doc/library/collections.rst | 4 +--- Lib/collections.py | 2 -- Lib/test/test_collections.py | 2 +- Misc/NEWS | 3 +++ 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 50115ab6857..ab290e79f35 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -628,9 +628,7 @@ Example: 'Return a new OrderedDict which maps field names to their values' return OrderedDict(zip(self._fields, self)) - __dict__ = property(_asdict) - - def _replace(_self, **kwds): + def _replace(_self, **kwds): 'Return a new Point object replacing specified fields with new values' result = _self._make(map(kwds.pop, ('x', 'y'), _self)) if kwds: diff --git a/Lib/collections.py b/Lib/collections.py index faf677b7bf7..af32e448a9c 100644 --- a/Lib/collections.py +++ b/Lib/collections.py @@ -259,8 +259,6 @@ def _asdict(self): 'Return a new OrderedDict which maps field names to their values' return OrderedDict(zip(self._fields, self)) - __dict__ = property(_asdict) - def _replace(_self, **kwds): 'Return a new {typename} object replacing specified fields with new values' result = _self._make(map(kwds.pop, {field_names!r}, _self)) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 313f81ff656..8bdeb3d8f1f 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -78,12 +78,12 @@ def test_instance(self): self.assertRaises(TypeError, eval, 'Point(XXX=1, y=2)', locals()) # wrong keyword argument self.assertRaises(TypeError, eval, 'Point(x=1)', locals()) # missing keyword argument self.assertEqual(repr(p), 'Point(x=11, y=22)') + self.assertNotIn('__dict__', dir(p)) # verify instance has no dict self.assertNotIn('__weakref__', dir(p)) self.assertEqual(p, Point._make([11, 22])) # test _make classmethod self.assertEqual(p._fields, ('x', 'y')) # test _fields attribute self.assertEqual(p._replace(x=1), (1, 22)) # test _replace method self.assertEqual(p._asdict(), dict(x=11, y=22)) # test _asdict method - self.assertEqual(vars(p), p._asdict()) # verify that vars() works try: p._replace(x=1, error=2) diff --git a/Misc/NEWS b/Misc/NEWS index b61bf230d18..625d300e842 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,9 @@ Build Core and Builtins ----------------- +- Issue #15535: Fixed regression in the pickling of named tuples by + removing the __dict__ property introduced in 2.7.4. + - Issue #17857: Prevent build failures with pre-3.5.0 versions of sqlite3, such as was shipped with Centos 5 and Mac OS X 10.4. From d0964230f2b72e7602702bde11270d9f372c5a1f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 4 May 2013 15:11:46 +0300 Subject: [PATCH 095/480] Issue #16316: mimetypes now recognizes the .xz and .txz (.tar.xz) extensions. --- Lib/mimetypes.py | 2 ++ Lib/test/test_mimetypes.py | 2 ++ Misc/NEWS | 2 ++ 3 files changed, 6 insertions(+) diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index ba4ea8257b6..18ade7381ad 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -386,12 +386,14 @@ def _default_mime_types(): '.taz': '.tar.gz', '.tz': '.tar.gz', '.tbz2': '.tar.bz2', + '.txz': '.tar.xz', } encodings_map = { '.gz': 'gzip', '.Z': 'compress', '.bz2': 'bzip2', + '.xz': 'xz', } # Before adding new types, make sure they are either registered with IANA, diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 3508b560568..788d7a8360a 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -21,6 +21,8 @@ def test_default_data(self): eq(self.db.guess_type("foo.tgz"), ("application/x-tar", "gzip")) eq(self.db.guess_type("foo.tar.gz"), ("application/x-tar", "gzip")) eq(self.db.guess_type("foo.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) + eq(self.db.guess_type("foo.tar.xz"), ("application/x-tar", "xz")) def test_data_urls(self): eq = self.assertEqual diff --git a/Misc/NEWS b/Misc/NEWS index 625d300e842..f4387940c61 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -34,6 +34,8 @@ Core and Builtins Library ------- +- Issue #16316: mimetypes now recognizes the .xz and .txz (.tar.xz) extensions. + - Issue #17192: Restore the patch for Issue #10309 which was ommitted in 2.7.4 when updating the bundled version of libffi used by ctypes. From 5bf680f005ab2362ab69d66a853b802f5b605f3a Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 4 May 2013 17:59:03 +0300 Subject: [PATCH 096/480] #7855: Add tests for ctypes/winreg for issues found in IronPython. Initial patch by Dino Viehland. --- Lib/ctypes/test/__init__.py | 2 +- Lib/ctypes/test/test_wintypes.py | 43 ++++++++++++++++++++++++++++++++ Lib/test/test_winreg.py | 5 ++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 5 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 Lib/ctypes/test/test_wintypes.py diff --git a/Lib/ctypes/test/__init__.py b/Lib/ctypes/test/__init__.py index 52230928ba5..808e4185ef0 100644 --- a/Lib/ctypes/test/__init__.py +++ b/Lib/ctypes/test/__init__.py @@ -62,7 +62,7 @@ def get_tests(package, mask, verbosity, exclude=()): continue try: mod = __import__(modname, globals(), locals(), ['*']) - except ResourceDenied, detail: + except (ResourceDenied, unittest.SkipTest) as detail: skipped.append(modname) if verbosity > 1: print >> sys.stderr, "Skipped %s: %s" % (modname, detail) diff --git a/Lib/ctypes/test/test_wintypes.py b/Lib/ctypes/test/test_wintypes.py new file mode 100644 index 00000000000..806fccef812 --- /dev/null +++ b/Lib/ctypes/test/test_wintypes.py @@ -0,0 +1,43 @@ +import sys +import unittest + +if not sys.platform.startswith('win'): + raise unittest.SkipTest('Windows-only test') + +from ctypes import * +from ctypes import wintypes + +class WinTypesTest(unittest.TestCase): + def test_variant_bool(self): + # reads 16-bits from memory, anything non-zero is True + for true_value in (1, 32767, 32768, 65535, 65537): + true = POINTER(c_int16)(c_int16(true_value)) + value = cast(true, POINTER(wintypes.VARIANT_BOOL)) + self.assertEqual(repr(value.contents), 'VARIANT_BOOL(True)') + + vb = wintypes.VARIANT_BOOL() + self.assertIs(vb.value, False) + vb.value = True + self.assertIs(vb.value, True) + vb.value = true_value + self.assertIs(vb.value, True) + + for false_value in (0, 65536, 262144, 2**33): + false = POINTER(c_int16)(c_int16(false_value)) + value = cast(false, POINTER(wintypes.VARIANT_BOOL)) + self.assertEqual(repr(value.contents), 'VARIANT_BOOL(False)') + + # allow any bool conversion on assignment to value + for set_value in (65536, 262144, 2**33): + vb = wintypes.VARIANT_BOOL() + vb.value = set_value + self.assertIs(vb.value, True) + + vb = wintypes.VARIANT_BOOL() + vb.value = [2, 3] + self.assertIs(vb.value, True) + vb.value = [] + self.assertIs(vb.value, False) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 683938acd45..e05d040998b 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -442,6 +442,11 @@ def test_disable_reflection(self): DeleteKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, KEY_WOW64_32KEY, 0) + def test_exception_numbers(self): + with self.assertRaises(WindowsError) as ctx: + QueryValue(HKEY_CLASSES_ROOT, 'some_value_that_does_not_exist') + + self.assertEqual(ctx.exception.errno, 2) def test_main(): test_support.run_unittest(LocalWinregTests, RemoteWinregTests, diff --git a/Misc/ACKS b/Misc/ACKS index 9f50b4c5dca..660f52ce145 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1050,6 +1050,7 @@ Jaap Vermeulen Al Vezza Jacques A. Vidrine John Viega +Dino Viehland Kannan Vijayan Kurt Vile Norman Vine diff --git a/Misc/NEWS b/Misc/NEWS index f4387940c61..628ad765386 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -81,6 +81,9 @@ Library Tests ----- +- Issue #7855: Add tests for ctypes/winreg for issues found in IronPython. + Initial patch by Dino Viehland. + - Issue #17712: Fix test_gdb failures on Ubuntu 13.04. - Issue #17065: Use process-unique key for winreg tests to avoid failures if From 2ea4f9833fa4c3e00aea7c0734ef104b553f0c9c Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 4 May 2013 20:18:34 +0200 Subject: [PATCH 097/480] Elaborate on bytes-like objects. --- Doc/glossary.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 0d02d1acfc8..219a6caf2ce 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -79,7 +79,11 @@ Glossary bytes-like object An object that supports the :ref:`buffer protocol `, - like :class:`str` or :class:`bytearray`. + like :class:`str`, :class:`bytearray` or :class:`memoryview`. + Bytes-like objects can be used for various operations that expect + binary data, such as compression, saving to a binary file or sending + over a socket. Some operations need the binary data to be mutable, + in which case not all bytes-like objects can apply. bytecode Python source code is compiled into bytecode, the internal representation From 5c9ed40fb86b56c75610e39d9da54cc2cb2bf4f9 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 4 May 2013 23:16:59 +0200 Subject: [PATCH 098/480] Issue #14173: Avoid crashing when reading a signal handler during interpreter shutdown. --- Misc/NEWS | 3 +++ Modules/signalmodule.c | 14 +++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 628ad765386..0691ad33956 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -34,6 +34,9 @@ Core and Builtins Library ------- +- Issue #14173: Avoid crashing when reading a signal handler during + interpreter shutdown. + - Issue #16316: mimetypes now recognizes the .xz and .txz (.tar.xz) extensions. - Issue #17192: Restore the patch for Issue #10309 which was ommitted diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index d5d628328b1..184b3a91a39 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -321,7 +321,10 @@ signal_signal(PyObject *self, PyObject *args) Handlers[sig_num].tripped = 0; Py_INCREF(obj); Handlers[sig_num].func = obj; - return old_handler; + if (old_handler != NULL) + return old_handler; + else + Py_RETURN_NONE; } PyDoc_STRVAR(signal_doc, @@ -349,8 +352,13 @@ signal_getsignal(PyObject *self, PyObject *args) return NULL; } old_handler = Handlers[sig_num].func; - Py_INCREF(old_handler); - return old_handler; + if (old_handler != NULL) { + Py_INCREF(old_handler); + return old_handler; + } + else { + Py_RETURN_NONE; + } } PyDoc_STRVAR(getsignal_doc, From 339e4f4e98a502613fd2af64b1c77b2d243b2b31 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 5 May 2013 22:36:09 +0300 Subject: [PATCH 099/480] #17883: Fix buildbot testing of Tkinter on Windows. Patch by Zachary Ware. --- Misc/NEWS | 3 +++ PCbuild/rt.bat | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 0691ad33956..1fe11be38a6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -84,6 +84,9 @@ Library Tests ----- +- Issue #17883: Fix buildbot testing of Tkinter on Windows. + Patch by Zachary Ware. + - Issue #7855: Add tests for ctypes/winreg for issues found in IronPython. Initial patch by Dino Viehland. diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index ee1661a21ab..9da1ed1b397 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -30,7 +30,7 @@ set prefix=.\ set suffix= set qmode= set dashO= -set tcltk= +set tcltk=tcltk :CheckOpts if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts @@ -38,7 +38,7 @@ if "%1"=="-q" (set qmode=yes) & shift & goto CheckOpts if "%1"=="-d" (set suffix=_d) & shift & goto CheckOpts if "%1"=="-x64" (set prefix=amd64) & (set tcltk=tcltk64) & shift & goto CheckOpts -PATH %PATH%;..\..\%tcltk%\bin +PATH %PATH%;%~dp0..\..\%tcltk%\bin set exe=%prefix%\python%suffix% set cmd=%exe% %dashO% -Wd -3 -E -tt ../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9 if defined qmode goto Qmode From 6758b386ade3206f2b64aae98599ed823a6d64b8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 5 May 2013 19:45:42 -0700 Subject: [PATCH 100/480] Issue 17862: Improve the signature of itertools grouper() recipe. Putting *n* after the *iterable* matches the signature of other itertools and recipes. Also, it reads better. Suggested by Ezio Melotti. --- Doc/library/itertools.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 0e98c1e2adf..76452811cde 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -732,9 +732,9 @@ which incur interpreter overhead. next(b, None) return izip(a, b) - def grouper(n, iterable, fillvalue=None): + def grouper(iterable, n, fillvalue=None): "Collect data into fixed-length chunks or blocks" - # grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx + # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx args = [iter(iterable)] * n return izip_longest(fillvalue=fillvalue, *args) From ae4f35544543ec3dcd29bd37106651159950ce30 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 5 May 2013 23:22:19 -0400 Subject: [PATCH 101/480] Issue17883: Update to assertIn to see why test fails on one buildbot. --- Lib/test/test_tcl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 8db11dc77d0..ea3d87d7b57 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -148,7 +148,7 @@ def testLoadWithUNC(self): env.unset("TCL_LIBRARY") f = os.popen('%s -c "import Tkinter; print Tkinter"' % (unc_name,)) - self.assertTrue('Tkinter.py' in f.read()) + self.assertIn('Tkinter.py', f.read()) # exit code must be zero self.assertEqual(f.close(), None) From 75a5e2a294bfb6a476fb709c552366cd8241d1f2 Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Mon, 6 May 2013 12:04:28 +0100 Subject: [PATCH 102/480] Issue #17805: Add AsyncResult alias for ApplyResult --- Lib/multiprocessing/pool.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 00c904a8a73..4d1860029c1 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -565,6 +565,8 @@ def _set(self, i, obj): self._cond.release() del self._cache[self._job] +AsyncResult = ApplyResult # create alias -- see #17805 + # # Class whose instances are returned by `Pool.map_async()` # From 2371d4ba906b1d2c1708e5b4521b4f94962ba234 Mon Sep 17 00:00:00 2001 From: Charles-Francois Natali Date: Mon, 6 May 2013 21:21:57 +0200 Subject: [PATCH 103/480] Issue #12181: select module: Fix struct kevent definition on OpenBSD 64-bit platforms. Patch by Federico Schwindt. --- Misc/NEWS | 3 +++ Modules/selectmodule.c | 35 ++++++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 1fe11be38a6..4c3f8333986 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -34,6 +34,9 @@ Core and Builtins Library ------- +- Issue #12181: select module: Fix struct kevent definition on OpenBSD 64-bit + platforms. Patch by Federico Schwindt. + - Issue #14173: Avoid crashing when reading a signal handler during interpreter shutdown. diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 650d9fe3fba..4cbf885b18f 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -1203,6 +1203,23 @@ static PyTypeObject kqueue_queue_Type; # error uintptr_t does not match int, long, or long long! #endif +/* + * kevent is not standard and its members vary across BSDs. + */ +#if !defined(__OpenBSD__) +# define IDENT_TYPE T_UINTPTRT +# define IDENT_CAST Py_intptr_t +# define DATA_TYPE T_INTPTRT +# define DATA_FMT_UNIT INTPTRT_FMT_UNIT +# define IDENT_AsType PyLong_AsUintptr_t +#else +# define IDENT_TYPE T_UINT +# define IDENT_CAST int +# define DATA_TYPE T_INT +# define DATA_FMT_UNIT "i" +# define IDENT_AsType PyLong_AsUnsignedLong +#endif + /* Unfortunately, we can't store python objects in udata, because * kevents in the kernel can be removed without warning, which would * forever lose the refcount on the object stored with it. @@ -1210,11 +1227,11 @@ static PyTypeObject kqueue_queue_Type; #define KQ_OFF(x) offsetof(kqueue_event_Object, x) static struct PyMemberDef kqueue_event_members[] = { - {"ident", T_UINTPTRT, KQ_OFF(e.ident)}, + {"ident", IDENT_TYPE, KQ_OFF(e.ident)}, {"filter", T_SHORT, KQ_OFF(e.filter)}, {"flags", T_USHORT, KQ_OFF(e.flags)}, {"fflags", T_UINT, KQ_OFF(e.fflags)}, - {"data", T_INTPTRT, KQ_OFF(e.data)}, + {"data", DATA_TYPE, KQ_OFF(e.data)}, {"udata", T_UINTPTRT, KQ_OFF(e.udata)}, {NULL} /* Sentinel */ }; @@ -1240,7 +1257,7 @@ kqueue_event_init(kqueue_event_Object *self, PyObject *args, PyObject *kwds) PyObject *pfd; static char *kwlist[] = {"ident", "filter", "flags", "fflags", "data", "udata", NULL}; - static char *fmt = "O|hhi" INTPTRT_FMT_UNIT UINTPTRT_FMT_UNIT ":kevent"; + static char *fmt = "O|hhi" DATA_FMT_UNIT UINTPTRT_FMT_UNIT ":kevent"; EV_SET(&(self->e), 0, EVFILT_READ, EV_ADD, 0, 0, 0); /* defaults */ @@ -1250,8 +1267,12 @@ kqueue_event_init(kqueue_event_Object *self, PyObject *args, PyObject *kwds) return -1; } - if (PyLong_Check(pfd)) { - self->e.ident = PyLong_AsUintptr_t(pfd); + if (PyLong_Check(pfd) +#if IDENT_TYPE == T_UINT + && PyLong_AsUnsignedLong(pfd) <= UINT_MAX +#endif + ) { + self->e.ident = IDENT_AsType(pfd); } else { self->e.ident = PyObject_AsFileDescriptor(pfd); @@ -1279,10 +1300,10 @@ kqueue_event_richcompare(kqueue_event_Object *s, kqueue_event_Object *o, Py_TYPE(s)->tp_name, Py_TYPE(o)->tp_name); return NULL; } - if (((result = s->e.ident - o->e.ident) == 0) && + if (((result = (IDENT_CAST)(s->e.ident - o->e.ident)) == 0) && ((result = s->e.filter - o->e.filter) == 0) && ((result = s->e.flags - o->e.flags) == 0) && - ((result = s->e.fflags - o->e.fflags) == 0) && + ((result = (int)(s->e.fflags - o->e.fflags)) == 0) && ((result = s->e.data - o->e.data) == 0) && ((result = s->e.udata - o->e.udata) == 0) ) { From 73929237423cc7219a1c4579aa46d352b6696433 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 6 May 2013 21:51:03 +0200 Subject: [PATCH 104/480] Issue #17289: The readline module now plays nicer with external modules or applications changing the rl_completer_word_break_characters global variable. Initial patch by Bradley Froehle. --- Misc/NEWS | 4 ++++ Modules/readline.c | 27 +++++++++++++++++++-------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 4c3f8333986..7989eb323be 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -34,6 +34,10 @@ Core and Builtins Library ------- +- Issue #17289: The readline module now plays nicer with external modules + or applications changing the rl_completer_word_break_characters global + variable. Initial patch by Bradley Froehle. + - Issue #12181: select module: Fix struct kevent definition on OpenBSD 64-bit platforms. Patch by Federico Schwindt. diff --git a/Modules/readline.c b/Modules/readline.c index b5e258db67e..233ebf95ad1 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -69,6 +69,10 @@ on_completion_display_matches_hook(char **matches, int num_matches, int max_length); +/* Memory allocated for rl_completer_word_break_characters + (see issue #17289 for the motivation). */ +static char *completer_word_break_characters; + /* Exported function to send one line to readline's init file parser */ static PyObject * @@ -344,12 +348,20 @@ set_completer_delims(PyObject *self, PyObject *args) { char *break_chars; - if(!PyArg_ParseTuple(args, "s:set_completer_delims", &break_chars)) { + if (!PyArg_ParseTuple(args, "s:set_completer_delims", &break_chars)) { return NULL; } - free((void*)rl_completer_word_break_characters); - rl_completer_word_break_characters = strdup(break_chars); - Py_RETURN_NONE; + /* Keep a reference to the allocated memory in the module state in case + some other module modifies rl_completer_word_break_characters + (see issue #17289). */ + free(completer_word_break_characters); + completer_word_break_characters = strdup(break_chars); + if (completer_word_break_characters) { + rl_completer_word_break_characters = completer_word_break_characters; + Py_RETURN_NONE; + } + else + return PyErr_NoMemory(); } PyDoc_STRVAR(doc_set_completer_delims, @@ -893,7 +905,8 @@ setup_readline(void) /* Set our completion function */ rl_attempted_completion_function = (CPPFunction *)flex_complete; /* Set Python word break characters */ - rl_completer_word_break_characters = + completer_word_break_characters = + rl_completer_word_break_characters = strdup(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?"); /* All nonalphanums except '.' */ @@ -906,7 +919,7 @@ setup_readline(void) */ #ifdef __APPLE__ if (using_libedit_emulation) - rl_read_init_file(NULL); + rl_read_init_file(NULL); else #endif /* __APPLE__ */ rl_initialize(); @@ -1137,8 +1150,6 @@ initreadline(void) if (m == NULL) return; - - PyOS_ReadlineFunctionPointer = call_readline; setup_readline(); } From 1a72e6330a1def7dac7234f1f5e01b7b72d1e18c Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 6 May 2013 22:19:48 +0200 Subject: [PATCH 105/480] Issue #17918: When using SSLSocket.accept(), if the SSL handshake failed on the new socket, the socket would linger indefinitely. Thanks to Peter Saveliev for reporting. --- Lib/ssl.py | 26 +++++++++++++++----------- Misc/NEWS | 4 ++++ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/Lib/ssl.py b/Lib/ssl.py index 88296358a02..329b9d10d64 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -344,17 +344,21 @@ def accept(self): SSL channel, and the address of the remote client.""" newsock, addr = socket.accept(self) - return (SSLSocket(newsock, - keyfile=self.keyfile, - certfile=self.certfile, - server_side=True, - cert_reqs=self.cert_reqs, - ssl_version=self.ssl_version, - ca_certs=self.ca_certs, - ciphers=self.ciphers, - do_handshake_on_connect=self.do_handshake_on_connect, - suppress_ragged_eofs=self.suppress_ragged_eofs), - addr) + try: + return (SSLSocket(newsock, + keyfile=self.keyfile, + certfile=self.certfile, + server_side=True, + cert_reqs=self.cert_reqs, + ssl_version=self.ssl_version, + ca_certs=self.ca_certs, + ciphers=self.ciphers, + do_handshake_on_connect=self.do_handshake_on_connect, + suppress_ragged_eofs=self.suppress_ragged_eofs), + addr) + except socket_error as e: + newsock.close() + raise e def makefile(self, mode='r', bufsize=-1): diff --git a/Misc/NEWS b/Misc/NEWS index 7989eb323be..7cef7ef01cb 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -34,6 +34,10 @@ Core and Builtins Library ------- +- Issue #17918: When using SSLSocket.accept(), if the SSL handshake failed + on the new socket, the socket would linger indefinitely. Thanks to + Peter Saveliev for reporting. + - Issue #17289: The readline module now plays nicer with external modules or applications changing the rl_completer_word_break_characters global variable. Initial patch by Bradley Froehle. From 761e15b468cb038330b71b093a4f58912986b224 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 6 May 2013 18:21:10 -0700 Subject: [PATCH 106/480] Issue 17920: Fix-up terminology in the set documentation --- Doc/library/stdtypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index d4bf4484825..76345e58f7f 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1849,8 +1849,8 @@ The constructors for both classes work the same: based on their members. For example, ``set('abc') == frozenset('abc')`` returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. - The subset and equality comparisons do not generalize to a complete ordering - function. For example, any two disjoint sets are not equal and are not + The subset and equality comparisons do not generalize to a total ordering + function. For example, any two non-empty disjoint sets are not equal and are not subsets of each other, so *all* of the following return ``False``: ``ab``. Accordingly, sets do not implement the :meth:`__cmp__` method. From 1d13041989a9cfe103a0847e43c9ebe2f66fbb52 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Tue, 7 May 2013 09:34:49 +0300 Subject: [PATCH 107/480] #17833: add debug output to investigate buildbot failure. --- Lib/test/test_tcl.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index ea3d87d7b57..a0e9e627ac6 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -4,6 +4,7 @@ import sys import os from test import test_support +from subprocess import Popen, PIPE # Skip this test if the _tkinter module wasn't built. _tkinter = test_support.import_module('_tkinter') @@ -146,11 +147,20 @@ def testLoadWithUNC(self): with test_support.EnvironmentVarGuard() as env: env.unset("TCL_LIBRARY") - f = os.popen('%s -c "import Tkinter; print Tkinter"' % (unc_name,)) + cmd = '%s -c "import Tkinter; print Tkinter"' % (unc_name,) + + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + out_data, err_data = p.communicate() + + msg = '\n\n'.join(['"Tkinter.py" not in output', + 'Command:', cmd, + 'stdout:', out_data, + 'stderr:', err_data]) + + self.assertIn('Tkinter.py', out_data, msg) + + self.assertEqual(p.wait(), 0, 'Non-zero exit code') - self.assertIn('Tkinter.py', f.read()) - # exit code must be zero - self.assertEqual(f.close(), None) def test_passing_values(self): def passValue(value): From 91b556e9312ebe187c071524404cf2c817f4a792 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Tue, 7 May 2013 11:14:27 +0300 Subject: [PATCH 108/480] #17714: document that the base64 codec adds a trailing newline. --- Doc/library/codecs.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index ce850f6c067..460af13d33e 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1113,7 +1113,9 @@ the table. | Codec | Aliases | Operand type | Purpose | +====================+===========================+================+===========================+ | base64_codec | base64, base-64 | byte string | Convert operand to MIME | -| | | | base64 | +| | | | base64 (the result always | +| | | | includes a trailing | +| | | | ``'\n'``) | +--------------------+---------------------------+----------------+---------------------------+ | bz2_codec | bz2 | byte string | Compress the operand | | | | | using bz2 | From edb18abd95a3cb5c915c7c471293a94727d8ecc2 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 8 May 2013 01:51:37 +0200 Subject: [PATCH 109/480] Issue #17926: Fix dbm.__contains__ on 64-bit big-endian machines. --- Misc/NEWS | 2 ++ Modules/dbmmodule.c | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 7cef7ef01cb..466781c1225 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -34,6 +34,8 @@ Core and Builtins Library ------- +- Issue #17926: Fix dbm.__contains__ on 64-bit big-endian machines. + - Issue #17918: When using SSLSocket.accept(), if the SSL handshake failed on the new socket, the socket would linger indefinitely. Thanks to Peter Saveliev for reporting. diff --git a/Modules/dbmmodule.c b/Modules/dbmmodule.c index f9c99a8b8f0..8b16def2676 100644 --- a/Modules/dbmmodule.c +++ b/Modules/dbmmodule.c @@ -168,11 +168,13 @@ static int dbm_contains(register dbmobject *dp, PyObject *v) { datum key, val; + char *ptr; + Py_ssize_t size; - if (PyString_AsStringAndSize(v, (char **)&key.dptr, - (Py_ssize_t *)&key.dsize)) { + if (PyString_AsStringAndSize(v, &ptr, &size)) return -1; - } + key.dptr = ptr; + key.dsize = size; /* Expand check_dbmobject_open to return -1 */ if (dp->di_dbm == NULL) { From 9dc629f1d81acf6154bad56c08e28cff2f7de20f Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 8 May 2013 02:07:13 +0200 Subject: [PATCH 110/480] Issue #17928: Fix test_structmembers on 64-bit big-endian machines. (_testcapi isn't Py_ssize_t-clean, the "s#" code should use an int for length) --- Misc/NEWS | 2 ++ Modules/_testcapimodule.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 466781c1225..579e09c5f70 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -97,6 +97,8 @@ Library Tests ----- +- Issue #17928: Fix test_structmembers on 64-bit big-endian machines. + - Issue #17883: Fix buildbot testing of Tkinter on Windows. Patch by Zachary Ware. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f1968e2a70c..b0386f0cace 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1813,7 +1813,7 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) ; test_structmembers *ob; const char *s = NULL; - Py_ssize_t string_len = 0; + int string_len = 0; ob = PyObject_New(test_structmembers, type); if (ob == NULL) return NULL; From c76afb7f915ec202ce09a2ca4d59b78de6bfc710 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 8 May 2013 03:23:10 +0200 Subject: [PATCH 111/480] Fix NEWS order --- Misc/NEWS | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 579e09c5f70..867ebc558a0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -6,14 +6,6 @@ What's New in Python 2.7.5? *Release date: XXXX-XX-XX* -Build ------ - -- Issue #17682: Add the _io module to Modules/Setup.dist (commented out). - -- Issue #17086: Search the include and library directories provided by the - compiler. - Core and Builtins ----------------- @@ -94,6 +86,14 @@ Library - Issue #17526: fix an IndexError raised while passing code without filename to inspect.findsource(). Initial patch by Tyler Doyle. +Build +----- + +- Issue #17682: Add the _io module to Modules/Setup.dist (commented out). + +- Issue #17086: Search the include and library directories provided by the + compiler. + Tests ----- From a39a05d1649b34c41cff8cc175bc7a691cf8b491 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Wed, 8 May 2013 10:53:11 +0300 Subject: [PATCH 112/480] #16523: improve attrgetter/itemgetter/methodcaller documentation. --- Doc/library/operator.rst | 47 ++++++++++++++++++++++++++++------------ Modules/operator.c | 14 ++++++------ 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 53d45b2306e..3a34978c617 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -490,13 +490,22 @@ lookups. These are useful for making fast field extractors as arguments for expect a function argument. -.. function:: attrgetter(attr[, args...]) +.. function:: attrgetter(attr) + attrgetter(*attrs) - Return a callable object that fetches *attr* from its operand. If more than one - attribute is requested, returns a tuple of attributes. After, - ``f = attrgetter('name')``, the call ``f(b)`` returns ``b.name``. After, - ``f = attrgetter('name', 'date')``, the call ``f(b)`` returns ``(b.name, - b.date)``. Equivalent to:: + Return a callable object that fetches *attr* from its operand. + If more than one attribute is requested, returns a tuple of attributes. + The attribute names can also contain dots. For example: + + * After ``f = attrgetter('name')``, the call ``f(b)`` returns ``b.name``. + + * After ``f = attrgetter('name', 'date')``, the call ``f(b)`` returns + ``(b.name, b.date)``. + + * After ``f = attrgetter('name.first', 'name.last')``, the call ``f(b)`` + returns ``(r.name.first, r.name.last)``. + + Equivalent to:: def attrgetter(*items): if len(items) == 1: @@ -514,8 +523,8 @@ expect a function argument. return obj - The attribute names can also contain dots; after ``f = attrgetter('date.month')``, - the call ``f(b)`` returns ``b.date.month``. +.. function:: itemgetter(item) + itemgetter(*items) .. versionadded:: 2.4 @@ -526,11 +535,16 @@ expect a function argument. Added support for dotted attributes. -.. function:: itemgetter(item[, args...]) - Return a callable object that fetches *item* from its operand using the operand's :meth:`__getitem__` method. If multiple items are specified, - returns a tuple of lookup values. Equivalent to:: + returns a tuple of lookup values. For example: + + * After ``f = itemgetter(2)``, the call ``f(r)`` returns ``r[2]``. + + * After ``g = itemgetter(2, 5, 3)``, the call ``g(r)`` returns + ``(r[2], r[5], r[3])``. + + Equivalent to:: def itemgetter(*items): if len(items) == 1: @@ -573,9 +587,14 @@ expect a function argument. Return a callable object that calls the method *name* on its operand. If additional arguments and/or keyword arguments are given, they will be given - to the method as well. After ``f = methodcaller('name')``, the call ``f(b)`` - returns ``b.name()``. After ``f = methodcaller('name', 'foo', bar=1)``, the - call ``f(b)`` returns ``b.name('foo', bar=1)``. Equivalent to:: + to the method as well. For example: + + * After ``f = methodcaller('name')``, the call ``f(b)`` returns ``b.name()``. + + * After ``f = methodcaller('name', 'foo', bar=1)``, the call ``f(b)`` + returns ``b.name('foo', bar=1)``. + + Equivalent to:: def methodcaller(name, *args, **kwargs): def caller(obj): diff --git a/Modules/operator.c b/Modules/operator.c index 274d8aad39f..3f005884426 100644 --- a/Modules/operator.c +++ b/Modules/operator.c @@ -412,8 +412,8 @@ PyDoc_STRVAR(itemgetter_doc, "itemgetter(item, ...) --> itemgetter object\n\ \n\ Return a callable object that fetches the given item(s) from its operand.\n\ -After, f=itemgetter(2), the call f(r) returns r[2].\n\ -After, g=itemgetter(2,5,3), the call g(r) returns (r[2], r[5], r[3])"); +After f = itemgetter(2), the call f(r) returns r[2].\n\ +After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3])"); static PyTypeObject itemgetter_type = { PyVarObject_HEAD_INIT(NULL, 0) @@ -592,9 +592,9 @@ PyDoc_STRVAR(attrgetter_doc, "attrgetter(attr, ...) --> attrgetter object\n\ \n\ Return a callable object that fetches the given attribute(s) from its operand.\n\ -After, f=attrgetter('name'), the call f(r) returns r.name.\n\ -After, g=attrgetter('name', 'date'), the call g(r) returns (r.name, r.date).\n\ -After, h=attrgetter('name.first', 'name.last'), the call h(r) returns\n\ +After f = attrgetter('name'), the call f(r) returns r.name.\n\ +After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date).\n\ +After h = attrgetter('name.first', 'name.last'), the call h(r) returns\n\ (r.name.first, r.name.last)."); static PyTypeObject attrgetter_type = { @@ -724,8 +724,8 @@ PyDoc_STRVAR(methodcaller_doc, "methodcaller(name, ...) --> methodcaller object\n\ \n\ Return a callable object that calls the given method on its operand.\n\ -After, f = methodcaller('name'), the call f(r) returns r.name().\n\ -After, g = methodcaller('name', 'date', foo=1), the call g(r) returns\n\ +After f = methodcaller('name'), the call f(r) returns r.name().\n\ +After g = methodcaller('name', 'date', foo=1), the call g(r) returns\n\ r.name('date', foo=1)."); static PyTypeObject methodcaller_type = { From 65bae8a383467e23b60afd6929b46e788702fe52 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 8 May 2013 21:52:31 +0300 Subject: [PATCH 113/480] Issue #17656: Skip test_extract_unicode_filenames if the FS encoding doesn't support non-ASCII filenames. --- Lib/test/test_zipfile.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index f960b6c7d4f..9e3a28b4383 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -18,7 +18,14 @@ from random import randint, random from unittest import skipUnless -from test.test_support import TESTFN, TESTFN_UNICODE, run_unittest, findfile, unlink +from test.test_support import TESTFN, TESTFN_UNICODE, TESTFN_ENCODING, \ + run_unittest, findfile, unlink +try: + TESTFN_UNICODE.encode(TESTFN_ENCODING) +except (UnicodeError, TypeError): + # Either the file system encoding is None, or the file name + # cannot be encoded in the file system encoding. + TESTFN_UNICODE = None TESTFN2 = TESTFN + "2" TESTFNDIR = TESTFN + "d" @@ -424,6 +431,7 @@ def check_file(self, filename, content): with open(filename, 'rb') as f: self.assertEqual(f.read(), content) + @skipUnless(TESTFN_UNICODE, "No Unicode filesystem semantics on this platform.") def test_extract_unicode_filenames(self): fnames = [u'foo.txt', os.path.basename(TESTFN_UNICODE)] content = 'Test for unicode filename' From 135d76893bdd9e00c32012f19771d124f0d8ce20 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Wed, 8 May 2013 23:42:41 -0400 Subject: [PATCH 114/480] Issue 16584: in filecomp._cmp, catch IOError as well as os.error. Patch by Till Maas. --- Lib/filecmp.py | 2 +- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/filecmp.py b/Lib/filecmp.py index 4728317fce9..3a793819c64 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -268,7 +268,7 @@ def cmpfiles(a, b, common, shallow=1): def _cmp(a, b, sh, abs=abs, cmp=cmp): try: return not abs(cmp(a, b, sh)) - except os.error: + except (os.error, IOError): return 2 diff --git a/Misc/ACKS b/Misc/ACKS index 660f52ce145..9e09abdf78a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -626,6 +626,7 @@ Mark Lutz Jim Lynch Mikael Lyngvig Martin von Löwis +Till Maas Jeff MacDonald John Machin Andrew I MacIntyre diff --git a/Misc/NEWS b/Misc/NEWS index 867ebc558a0..6eea7de4182 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -26,6 +26,9 @@ Core and Builtins Library ------- +- Issue 16584: in filecomp._cmp, catch IOError as well as os.error. + Patch by Till Maas. + - Issue #17926: Fix dbm.__contains__ on 64-bit big-endian machines. - Issue #17918: When using SSLSocket.accept(), if the SSL handshake failed From 66772d1a5d57d609041b3e47616914a6dd702c4f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 9 May 2013 14:22:05 +0300 Subject: [PATCH 115/480] Issue #16601: Restarting iteration over tarfile no more continues from where it left off. Patch by Michael Birtwell. --- Lib/tarfile.py | 12 +++++++----- Lib/test/test_tarfile.py | 8 ++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 89d8cc3ffd5..16a6e86dce1 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2462,16 +2462,18 @@ def next(self): # Fix for SF #1100429: Under rare circumstances it can # happen that getmembers() is called during iteration, # which will cause TarIter to stop prematurely. - if not self.tarfile._loaded: + + if self.index == 0 and self.tarfile.firstmember is not None: + tarinfo = self.tarfile.next() + elif self.index < len(self.tarfile.members): + tarinfo = self.tarfile.members[self.index] + elif not self.tarfile._loaded: tarinfo = self.tarfile.next() if not tarinfo: self.tarfile._loaded = True raise StopIteration else: - try: - tarinfo = self.tarfile.members[self.index] - except IndexError: - raise StopIteration + raise StopIteration self.index += 1 return tarinfo diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index af59e277f3e..69d342a5031 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -345,6 +345,14 @@ def test_init_close_fobj(self): finally: os.remove(empty) + def test_parallel_iteration(self): + # Issue #16601: Restarting iteration over tarfile continued + # from where it left off. + with tarfile.open(self.tarname) as tar: + for m1, m2 in zip(tar, tar): + self.assertEqual(m1.offset, m2.offset) + self.assertEqual(m1.name, m2.name) + class StreamReadTest(CommonReadTest): diff --git a/Misc/ACKS b/Misc/ACKS index 9e09abdf78a..7f78dbdeb7b 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -96,6 +96,7 @@ Natalia B. Bidart David Binger Dominic Binks Philippe Biondi +Michael Birtwell Stuart Bishop Roy Bixler Jonathan Black diff --git a/Misc/NEWS b/Misc/NEWS index 6eea7de4182..2c094208c63 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -26,6 +26,9 @@ Core and Builtins Library ------- +- Issue #16601: Restarting iteration over tarfile no more continues from where + it left off. Patch by Michael Birtwell. + - Issue 16584: in filecomp._cmp, catch IOError as well as os.error. Patch by Till Maas. From 461779b5e93e26419fe3e62210a0401f1280d973 Mon Sep 17 00:00:00 2001 From: Andrew Kuchling Date: Thu, 9 May 2013 20:55:22 -0400 Subject: [PATCH 116/480] #14878: add cross-reference to the yield statement. (Backported from 3.x by Jan Duzinkiewicz.) --- Doc/reference/simple_stmts.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index d8cfd5c6ca5..a6fd0649a91 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -511,6 +511,9 @@ reference count or by being garbage collected), the generator-iterator's :meth:`close` method will be called, allowing any pending :keyword:`finally` clauses to execute. +For full details of :keyword:`yield` semantics, refer to the :ref:`yieldexpr` +section. + .. note:: In Python 2.2, the :keyword:`yield` statement was only allowed when the From e68b1687792426814a2ed9ddc3fe4853abc5b57c Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Fri, 10 May 2013 19:57:04 +0300 Subject: [PATCH 117/480] #17949: fix merge glitch in itemgetter signature. Patch by Martijn Pieters. --- Doc/library/operator.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 3a34978c617..9f17e987675 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -523,9 +523,6 @@ expect a function argument. return obj -.. function:: itemgetter(item) - itemgetter(*items) - .. versionadded:: 2.4 .. versionchanged:: 2.5 @@ -535,6 +532,9 @@ expect a function argument. Added support for dotted attributes. +.. function:: itemgetter(item) + itemgetter(*items) + Return a callable object that fetches *item* from its operand using the operand's :meth:`__getitem__` method. If multiple items are specified, returns a tuple of lookup values. For example: From d4e4ff1395a17c3fa5fa1c27113e6e5094e9166d Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 11 May 2013 13:00:05 -0500 Subject: [PATCH 118/480] -Wformat is needed by gcc 4.8 (closes #17547) --- Misc/NEWS | 3 +++ configure | 2 +- configure.ac | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 2c094208c63..3e177ca834a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -95,6 +95,9 @@ Library Build ----- +- Issue #17547: In configure, explicitly pass -Wformat for the benefit for GCC + 4.8. + - Issue #17682: Add the _io module to Modules/Setup.dist (commented out). - Issue #17086: Search the include and library directories provided by the diff --git a/configure b/configure index 528aa512ae0..dc0dfd04ad1 100755 --- a/configure +++ b/configure @@ -6253,7 +6253,7 @@ then { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether gcc supports ParseTuple __format__" >&5 $as_echo_n "checking whether gcc supports ParseTuple __format__... " >&6; } save_CFLAGS=$CFLAGS - CFLAGS="$CFLAGS -Werror" + CFLAGS="$CFLAGS -Werror -Wformat" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ diff --git a/configure.ac b/configure.ac index cb2a958fbfe..30f5bf404a3 100644 --- a/configure.ac +++ b/configure.ac @@ -1326,7 +1326,7 @@ if test "$GCC" = "yes" then AC_MSG_CHECKING(whether gcc supports ParseTuple __format__) save_CFLAGS=$CFLAGS - CFLAGS="$CFLAGS -Werror" + CFLAGS="$CFLAGS -Werror -Wformat" AC_COMPILE_IFELSE([ AC_LANG_PROGRAM([[void f(char*,...)__attribute((format(PyArg_ParseTuple, 1, 2)));]], [[]]) ],[ From be3ef8bb0f75bccc0bac11e28449e6bf1d813fce Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 11 May 2013 13:17:13 -0500 Subject: [PATCH 119/480] backout 214d8909513d for regressions (#1159051) --- Lib/gzip.py | 69 ++++++++++++++++++++++--------------------- Lib/test/test_bz2.py | 18 ----------- Lib/test/test_gzip.py | 17 ----------- Misc/NEWS | 3 -- 4 files changed, 36 insertions(+), 71 deletions(-) diff --git a/Lib/gzip.py b/Lib/gzip.py index 99b2d3ef078..a613bae876d 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -21,6 +21,9 @@ def write32u(output, value): # or unsigned. output.write(struct.pack(" self.extrasize: - if not self._read(readsize): - if size > self.extrasize: - size = self.extrasize - break - readsize = min(self.max_read_chunk, readsize * 2) + try: + while size > self.extrasize: + self._read(readsize) + readsize = min(self.max_read_chunk, readsize * 2) + except EOFError: + if size > self.extrasize: + size = self.extrasize offset = self.offset - self.extrastart chunk = self.extrabuf[offset: offset + size] @@ -275,7 +277,7 @@ def _unread(self, buf): def _read(self, size=1024): if self.fileobj is None: - return False + raise EOFError, "Reached EOF" if self._new_member: # If the _new_member flag is set, we have to @@ -286,7 +288,7 @@ def _read(self, size=1024): pos = self.fileobj.tell() # Save current position self.fileobj.seek(0, 2) # Seek to end of file if pos == self.fileobj.tell(): - return False + raise EOFError, "Reached EOF" else: self.fileobj.seek( pos ) # Return to original position @@ -303,10 +305,9 @@ def _read(self, size=1024): if buf == "": uncompress = self.decompress.flush() - self.fileobj.seek(-len(self.decompress.unused_data), 1) self._read_eof() self._add_read_data( uncompress ) - return False + raise EOFError, 'Reached EOF' uncompress = self.decompress.decompress(buf) self._add_read_data( uncompress ) @@ -316,14 +317,13 @@ def _read(self, size=1024): # so seek back to the start of the unused data, finish up # this member, and read a new gzip header. # (The number of bytes to seek back is the length of the unused - # data) - self.fileobj.seek(-len(self.decompress.unused_data), 1) + # data, minus 8 because _read_eof() will rewind a further 8 bytes) + self.fileobj.seek( -len(self.decompress.unused_data)+8, 1) # Check the CRC and file size, and set the flag so we read # a new member on the next call self._read_eof() self._new_member = True - return True def _add_read_data(self, data): self.crc = zlib.crc32(data, self.crc) & 0xffffffffL @@ -334,11 +334,14 @@ def _add_read_data(self, data): self.size = self.size + len(data) def _read_eof(self): - # We've read to the end of the file. + # We've read to the end of the file, so we have to rewind in order + # to reread the 8 bytes containing the CRC and the file size. # We check the that the computed CRC and size of the # uncompressed data matches the stored values. Note that the size # stored is the true file size mod 2**32. - crc32, isize = struct.unpack(" Date: Sat, 11 May 2013 22:24:28 -0500 Subject: [PATCH 120/480] prevent IDLE from trying to close when sys.stdin is reassigned (#17838) --- Lib/idlelib/run.py | 5 +++++ Misc/NEWS | 2 ++ 2 files changed, 7 insertions(+) diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index e82ebf4c92a..9cc009fb7b8 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -264,6 +264,11 @@ def handle(self): IOBinding.encoding) sys.stderr = PyShell.PseudoOutputFile(self.console, "stderr", IOBinding.encoding) + + # Keep a reference to stdin so that it won't try to exit IDLE if + # sys.stdin gets changed from within IDLE's shell. See issue17838. + self._keep_stdin = sys.stdin + self.interp = self.get_remote_proxy("interp") rpc.RPCHandler.getresponse(self, myseq=None, wait=0.05) diff --git a/Misc/NEWS b/Misc/NEWS index 624a8afb577..0405fb99e77 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -122,6 +122,8 @@ Tests IDLE ---- +- Issue #17838: Allow sys.stdin to be reassigned. + - Issue #14735: Update IDLE docs to omit "Control-z on Windows". - Issue #17585: Fixed IDLE regression. Now closes when using exit() or quit(). From 2b647f60a129e658931731044bb783e4c3b69c56 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 11 May 2013 22:29:20 -0500 Subject: [PATCH 121/480] bump version to 2.7.5 --- Include/patchlevel.h | 4 ++-- Lib/distutils/__init__.py | 2 +- Lib/idlelib/idlever.py | 2 +- Misc/NEWS | 2 +- Misc/RPM/python-2.7.spec | 2 +- README | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index b62d890bdc8..bdf179e6241 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -22,12 +22,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 2 #define PY_MINOR_VERSION 7 -#define PY_MICRO_VERSION 4 +#define PY_MICRO_VERSION 5 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "2.7.4+" +#define PY_VERSION "2.7.5" /*--end constants--*/ /* Subversion Revision number of this file (not of the repository). Empty diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py index 84dbc3806a7..98f4bb3a49a 100644 --- a/Lib/distutils/__init__.py +++ b/Lib/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.4" +__version__ = "2.7.5" #--end constants-- diff --git a/Lib/idlelib/idlever.py b/Lib/idlelib/idlever.py index 77421d5f770..481afab24cb 100644 --- a/Lib/idlelib/idlever.py +++ b/Lib/idlelib/idlever.py @@ -1 +1 @@ -IDLE_VERSION = "2.7.4" +IDLE_VERSION = "2.7.5" diff --git a/Misc/NEWS b/Misc/NEWS index 0405fb99e77..8f9739022e9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -4,7 +4,7 @@ Python News What's New in Python 2.7.5? =========================== -*Release date: XXXX-XX-XX* +*Release date: 2013-05-12* Core and Builtins ----------------- diff --git a/Misc/RPM/python-2.7.spec b/Misc/RPM/python-2.7.spec index c00ba96cbd6..9df3b2c88e9 100644 --- a/Misc/RPM/python-2.7.spec +++ b/Misc/RPM/python-2.7.spec @@ -39,7 +39,7 @@ %define name python #--start constants-- -%define version 2.7.4 +%define version 2.7.5 %define libvers 2.7 #--end constants-- %define release 1pydotorg diff --git a/README b/README index 9b3f16eea49..014028dacbf 100644 --- a/README +++ b/README @@ -1,4 +1,4 @@ -This is Python version 2.7.4 +This is Python version 2.7.5 ============================ Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, From 2c8e18fd24a39d15faa87d142e19b4031ad31826 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 11 May 2013 22:31:33 -0500 Subject: [PATCH 122/480] update topics --- Lib/pydoc_data/topics.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 75a71ff85cb..eb1c15e7180 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,7 +1,7 @@ -# Autogenerated by Sphinx on Sat Apr 6 09:55:30 2013 +# Autogenerated by Sphinx on Sat May 11 22:31:13 2013 topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, ``assert expression``, is equivalent to\n\n if __debug__:\n if not expression: raise AssertionError\n\nThe extended form, ``assert expression1, expression2``, is equivalent\nto\n\n if __debug__:\n if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that ``__debug__`` and ``AssertionError``\nrefer to the built-in variables with those names. In the current\nimplementation, the built-in variable ``__debug__`` is ``True`` under\nnormal circumstances, ``False`` when optimization is requested\n(command line option -O). The current code generator emits no code\nfor an assert statement when optimization is requested at compile\ntime. Note that it is unnecessary to include the source code for the\nexpression that failed in the error message; it will be displayed as\npart of the stack trace.\n\nAssignments to ``__debug__`` are illegal. The value for the built-in\nvariable is determined when the interpreter starts.\n', 'assignment': '\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list is recursively defined as\nfollows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` statement in the\n current code block: the name is bound to the object in the current\n local namespace.\n\n * Otherwise: the name is bound to the object in the current global\n namespace.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield a plain integer. If it is negative, the\n sequence\'s length is added to it. The resulting value must be a\n nonnegative integer less than the sequence\'s length, and the\n sequence is asked to assign the assigned object to its item with\n that index. If the index is out of range, ``IndexError`` is raised\n (assignment to a subscripted sequence cannot add new items to a\n list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to (small) integers. If either\n bound is negative, the sequence\'s length is added to it. The\n resulting bounds are clipped to lie between zero and the sequence\'s\n length, inclusive. Finally, the sequence object is asked to replace\n the slice with the items of the assigned sequence. The length of\n the slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print x\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', - 'atom-identifiers': '\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name in front of the name, with leading underscores removed, and\na single underscore inserted in front of the class name. For example,\nthe identifier ``__spam`` occurring in a class named ``Ham`` will be\ntransformed to ``_Ham__spam``. This transformation is independent of\nthe syntactical context in which the identifier is used. If the\ntransformed name is extremely long (longer than 255 characters),\nimplementation defined truncation may happen. If the class name\nconsists only of underscores, no transformation is done.\n', + 'atom-identifiers': '\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name, with leading underscores removed and a single underscore\ninserted, in front of the name. For example, the identifier\n``__spam`` occurring in a class named ``Ham`` will be transformed to\n``_Ham__spam``. This transformation is independent of the syntactical\ncontext in which the identifier is used. If the transformed name is\nextremely long (longer than 255 characters), implementation defined\ntruncation may happen. If the class name consists only of underscores,\nno transformation is done.\n', 'atom-literals': "\nLiterals\n********\n\nPython supports string literals and various numeric literals:\n\n literal ::= stringliteral | integer | longinteger\n | floatnumber | imagnumber\n\nEvaluation of a literal yields an object of the given type (string,\ninteger, long integer, floating point number, complex number) with the\ngiven value. The value may be approximated in the case of floating\npoint and imaginary (complex) literals. See section *Literals* for\ndetails.\n\nAll literals correspond to immutable data types, and hence the\nobject's identity is less important than its value. Multiple\nevaluations of literals with the same value (either the same\noccurrence in the program text or a different occurrence) may obtain\nthe same object or a different object with the same value.\n", 'attribute-access': '\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n===========================================\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n', 'attribute-references': '\nAttribute references\n********************\n\nAn attribute reference is a primary followed by a period and a name:\n\n attributeref ::= primary "." identifier\n\nThe primary must evaluate to an object of a type that supports\nattribute references, e.g., a module, list, or an instance. This\nobject is then asked to produce the attribute whose name is the\nidentifier. If this attribute is not available, the exception\n``AttributeError`` is raised. Otherwise, the type and value of the\nobject produced is determined by the object. Multiple evaluations of\nthe same attribute reference may yield different objects.\n', @@ -33,14 +33,14 @@ 'exprlists': '\nExpression lists\n****************\n\n expression_list ::= expression ( "," expression )* [","]\n\nAn expression list containing at least one comma yields a tuple. The\nlength of the tuple is the number of expressions in the list. The\nexpressions are evaluated from left to right.\n\nThe trailing comma is required only to create a single tuple (a.k.a. a\n*singleton*); it is optional in all other cases. A single expression\nwithout a trailing comma doesn\'t create a tuple, but rather yields the\nvalue of that expression. (To create an empty tuple, use an empty pair\nof parentheses: ``()``.)\n', 'floating': '\nFloating point literals\n***********************\n\nFloating point literals are described by the following lexical\ndefinitions:\n\n floatnumber ::= pointfloat | exponentfloat\n pointfloat ::= [intpart] fraction | intpart "."\n exponentfloat ::= (intpart | pointfloat) exponent\n intpart ::= digit+\n fraction ::= "." digit+\n exponent ::= ("e" | "E") ["+" | "-"] digit+\n\nNote that the integer and exponent parts of floating point numbers can\nlook like octal integers, but are interpreted using radix 10. For\nexample, ``077e010`` is legal, and denotes the same number as\n``77e10``. The allowed range of floating point literals is\nimplementation-dependent. Some examples of floating point literals:\n\n 3.14 10. .001 1e100 3.14e-10 0e0\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator ``-`` and the\nliteral ``1``.\n', 'for': '\nThe ``for`` statement\n*********************\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed. When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop. Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An internal\n counter is used to keep track of which item is used next, and this\n is incremented on each iteration. When this counter has reached the\n length of the sequence the loop terminates. This means that if the\n suite deletes the current (or a previous) item from the sequence,\n the next item will be skipped (since it gets the index of the\n current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n', - 'formatstrings': '\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= +\n conversion ::= "r" | "s"\n format_spec ::= \n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``. These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either a\nnumber or a keyword. If it\'s a number, it refers to a positional\nargument, and if it\'s a keyword, it refers to a named keyword\nargument. If the numerical arg_names in a format string are 0, 1, 2,\n... in sequence, they can all be omitted (not just some) and the\nnumbers 0, 1, 2, ... will be automatically inserted in that order.\nBecause *arg_name* is not quote-delimited, it is not possible to\nspecify arbitrary dictionary keys (e.g., the strings ``\'10\'`` or\n``\':-]\'``) within a format string. The *arg_name* can be followed by\nany number of index or attribute expressions. An expression of the\nform ``\'.name\'`` selects the named attribute using ``getattr()``,\nwhile an expression of the form ``\'[index]\'`` does an index lookup\nusing ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself. However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting. By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in ``format()`` function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= \n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'{\' or \'}\'. The\npresence of a fill character is signaled by the character following\nit, which must be one of the alignment options. If the second\ncharacter of *format_spec* is not a valid alignment option, then it is\nassumed that both the fill character and the alignment option are\nabsent.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'<\'`` | Forces the field to be left-aligned within the available |\n | | space (this is the default for most objects). |\n +-----------+------------------------------------------------------------+\n | ``\'>\'`` | Forces the field to be right-aligned within the available |\n | | space (this is the default for numbers). |\n +-----------+------------------------------------------------------------+\n | ``\'=\'`` | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | ``\'^\'`` | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'+\'`` | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | ``\'-\'`` | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output. If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nPreceding the *width* field by a zero (``\'0\'``) character enables\nsign-aware zero-padding for numeric types. This is equivalent to a\n*fill* character of ``\'0\'`` with an *alignment* type of ``\'=\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'s\'`` | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'s\'``. |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'b\'`` | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | ``\'c\'`` | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | ``\'d\'`` | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | ``\'o\'`` | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | ``\'x\'`` | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'X\'`` | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'d\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'d\'``. |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'e\'`` | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n +-----------+------------------------------------------------------------+\n | ``\'E\'`` | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | ``\'f\'`` | Fixed point. Displays the number as a fixed-point number. |\n +-----------+------------------------------------------------------------+\n | ``\'F\'`` | Fixed point. Same as ``\'f\'``. |\n +-----------+------------------------------------------------------------+\n | ``\'g\'`` | General format. For a given precision ``p >= 1``, this |\n | | rounds the number to ``p`` significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1`` would have exponent ``exp``. Then if ``-4 <= exp |\n | | < p``, the number is formatted with presentation type |\n | | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number |\n | | is formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1``. In both cases insignificant trailing zeros are |\n | | removed from the significand, and the decimal point is |\n | | also removed if there are no remaining digits following |\n | | it. Positive and negative infinity, positive and negative |\n | | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n | | ``-0`` and ``nan`` respectively, regardless of the |\n | | precision. A precision of ``0`` is treated as equivalent |\n | | to a precision of ``1``. |\n +-----------+------------------------------------------------------------+\n | ``\'G\'`` | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n | | if the number gets too large. The representations of |\n | | infinity and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'g\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | ``\'%\'`` | Percentage. Multiplies the number by 100 and displays in |\n | | fixed (``\'f\'``) format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'g\'``. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 2.7+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point(object):\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19.5\n >>> total = 22\n >>> \'Correct answers: {:.2%}\'.format(points/total)\n \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12):\n ... for base in \'dXob\':\n ... print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n ... print\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', + 'formatstrings': '\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= +\n conversion ::= "r" | "s"\n format_spec ::= \n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``. These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either a\nnumber or a keyword. If it\'s a number, it refers to a positional\nargument, and if it\'s a keyword, it refers to a named keyword\nargument. If the numerical arg_names in a format string are 0, 1, 2,\n... in sequence, they can all be omitted (not just some) and the\nnumbers 0, 1, 2, ... will be automatically inserted in that order.\nBecause *arg_name* is not quote-delimited, it is not possible to\nspecify arbitrary dictionary keys (e.g., the strings ``\'10\'`` or\n``\':-]\'``) within a format string. The *arg_name* can be followed by\nany number of index or attribute expressions. An expression of the\nform ``\'.name\'`` selects the named attribute using ``getattr()``,\nwhile an expression of the form ``\'[index]\'`` does an index lookup\nusing ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself. However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting. By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in ``format()`` function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= \n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'{\' or \'}\'. The\npresence of a fill character is signaled by the character following\nit, which must be one of the alignment options. If the second\ncharacter of *format_spec* is not a valid alignment option, then it is\nassumed that both the fill character and the alignment option are\nabsent.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'<\'`` | Forces the field to be left-aligned within the available |\n | | space (this is the default for most objects). |\n +-----------+------------------------------------------------------------+\n | ``\'>\'`` | Forces the field to be right-aligned within the available |\n | | space (this is the default for numbers). |\n +-----------+------------------------------------------------------------+\n | ``\'=\'`` | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | ``\'^\'`` | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'+\'`` | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | ``\'-\'`` | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output. If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nPreceding the *width* field by a zero (``\'0\'``) character enables\nsign-aware zero-padding for numeric types. This is equivalent to a\n*fill* character of ``\'0\'`` with an *alignment* type of ``\'=\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'s\'`` | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'s\'``. |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'b\'`` | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | ``\'c\'`` | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | ``\'d\'`` | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | ``\'o\'`` | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | ``\'x\'`` | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'X\'`` | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'d\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'d\'``. |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'e\'`` | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n | | The default precision is ``6``. |\n +-----------+------------------------------------------------------------+\n | ``\'E\'`` | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | ``\'f\'`` | Fixed point. Displays the number as a fixed-point number. |\n | | The default precision is ``6``. |\n +-----------+------------------------------------------------------------+\n | ``\'F\'`` | Fixed point. Same as ``\'f\'``. |\n +-----------+------------------------------------------------------------+\n | ``\'g\'`` | General format. For a given precision ``p >= 1``, this |\n | | rounds the number to ``p`` significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1`` would have exponent ``exp``. Then if ``-4 <= exp |\n | | < p``, the number is formatted with presentation type |\n | | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number |\n | | is formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1``. In both cases insignificant trailing zeros are |\n | | removed from the significand, and the decimal point is |\n | | also removed if there are no remaining digits following |\n | | it. Positive and negative infinity, positive and negative |\n | | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n | | ``-0`` and ``nan`` respectively, regardless of the |\n | | precision. A precision of ``0`` is treated as equivalent |\n | | to a precision of ``1``. The default precision is ``6``. |\n +-----------+------------------------------------------------------------+\n | ``\'G\'`` | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n | | if the number gets too large. The representations of |\n | | infinity and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'g\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | ``\'%\'`` | Percentage. Multiplies the number by 100 and displays in |\n | | fixed (``\'f\'``) format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'g\'``. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 2.7+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point(object):\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19.5\n >>> total = 22\n >>> \'Correct answers: {:.2%}\'.format(points/total)\n \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12):\n ... for base in \'dXob\':\n ... print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n ... print\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', 'function': '\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n decorated ::= decorators (classdef | funcdef)\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n funcdef ::= "def" funcname "(" [parameter_list] ")" ":" suite\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" identifier ["," "**" identifier]\n | "**" identifier\n | defparameter [","] )\n defparameter ::= parameter ["=" expression]\n sublist ::= parameter ("," parameter)* [","]\n parameter ::= identifier | "(" sublist ")"\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to:\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more top-level *parameters* have the form *parameter*\n``=`` *expression*, the function is said to have "default parameter\nvalues." For a parameter with a default value, the corresponding\n*argument* may be omitted from a call, in which case the parameter\'s\ndefault value is substituted. If a parameter has a default value, all\nfollowing parameters must also have a default value --- this is a\nsyntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that the same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n', 'global': '\nThe ``global`` statement\n************************\n\n global_stmt ::= "global" identifier ("," identifier)*\n\nThe ``global`` statement is a declaration which holds for the entire\ncurrent code block. It means that the listed identifiers are to be\ninterpreted as globals. It would be impossible to assign to a global\nvariable without ``global``, although free variables may refer to\nglobals without being declared global.\n\nNames listed in a ``global`` statement must not be used in the same\ncode block textually preceding that ``global`` statement.\n\nNames listed in a ``global`` statement must not be defined as formal\nparameters or in a ``for`` loop control target, ``class`` definition,\nfunction definition, or ``import`` statement.\n\n**CPython implementation detail:** The current implementation does not\nenforce the latter two restrictions, but programs should not abuse\nthis freedom, as future implementations may enforce them or silently\nchange the meaning of the program.\n\n**Programmer\'s note:** the ``global`` is a directive to the parser.\nIt applies only to code parsed at the same time as the ``global``\nstatement. In particular, a ``global`` statement contained in an\n``exec`` statement does not affect the code block *containing* the\n``exec`` statement, and code contained in an ``exec`` statement is\nunaffected by ``global`` statements in the code containing the\n``exec`` statement. The same applies to the ``eval()``,\n``execfile()`` and ``compile()`` functions.\n', 'id-classes': '\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of ``__*__`` names, in any context, that does\n not follow explicitly documented use, is subject to breakage\n without warning.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', 'identifiers': '\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions:\n\n identifier ::= (letter|"_") (letter | digit | "_")*\n letter ::= lowercase | uppercase\n lowercase ::= "a"..."z"\n uppercase ::= "A"..."Z"\n digit ::= "0"..."9"\n\nIdentifiers are unlimited in length. Case is significant.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers. They must\nbe spelled exactly as written here:\n\n and del from not while\n as elif global or with\n assert else if pass yield\n break except import print\n class exec in raise\n continue finally is return\n def for lambda try\n\nChanged in version 2.4: ``None`` became a constant and is now\nrecognized by the compiler as a name for the built-in object ``None``.\nAlthough it is not a keyword, you cannot assign a different object to\nit.\n\nChanged in version 2.5: Using ``as`` and ``with`` as identifiers\ntriggers a warning. To use them as keywords, enable the\n``with_statement`` future feature .\n\nChanged in version 2.6: ``as`` and ``with`` are full keywords.\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of ``__*__`` names, in any context, that does\n not follow explicitly documented use, is subject to breakage\n without warning.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', 'if': '\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n', 'imaginary': '\nImaginary literals\n******************\n\nImaginary literals are described by the following lexical definitions:\n\n imagnumber ::= (floatnumber | intpart) ("j" | "J")\n\nAn imaginary literal yields a complex number with a real part of 0.0.\nComplex numbers are represented as a pair of floating point numbers\nand have the same restrictions on their range. To create a complex\nnumber with a nonzero real part, add a floating point number to it,\ne.g., ``(3+4j)``. Some examples of imaginary literals:\n\n 3.14j 10.j 10j .001j 1e100j 3.14e-10j\n', - 'import': '\nThe ``import`` statement\n************************\n\n import_stmt ::= "import" module ["as" name] ( "," module ["as" name] )*\n | "from" relative_module "import" identifier ["as" name]\n ( "," identifier ["as" name] )*\n | "from" relative_module "import" "(" identifier ["as" name]\n ( "," identifier ["as" name] )* [","] ")"\n | "from" module "import" "*"\n module ::= (identifier ".")* identifier\n relative_module ::= "."* module | "."+\n name ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns *None`* is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any. If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound. As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname". If a name is not\nfound, ``ImportError`` is raised. If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement..\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module. The names given in ``__all__`` are all considered public\nand are required to exist. If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope. If the\nwild card form of import --- ``import *`` --- is used in a function\nand the function contains or is a nested block with free variables,\nthe compiler will raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimport mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python. The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language. It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n future_statement ::= "from" "__future__" "import" feature ["as" name]\n ("," feature ["as" name])*\n | "from" "__future__" "import" "(" feature ["as" name]\n ("," feature ["as" name])* [","] ")"\n feature ::= identifier\n name ::= identifier\n\nA future statement must appear near the top of the module. The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 2.6 are ``unicode_literals``,\n``print_function``, ``absolute_import``, ``division``, ``generators``,\n``nested_scopes`` and ``with_statement``. ``generators``,\n``with_statement``, ``nested_scopes`` are redundant in Python version\n2.6 and above because they are always enabled.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code. It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently. Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by an ``exec`` statement or calls to the built-in\nfunctions ``compile()`` and ``execfile()`` that occur in a module\n``M`` containing a future statement will, by default, use the new\nsyntax or semantics associated with the future statement. This can,\nstarting with Python 2.2 be controlled by optional arguments to\n``compile()`` --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session. If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n **PEP 236** - Back to the __future__\n The original proposal for the __future__ mechanism.\n', + 'import': '\nThe ``import`` statement\n************************\n\n import_stmt ::= "import" module ["as" name] ( "," module ["as" name] )*\n | "from" relative_module "import" identifier ["as" name]\n ( "," identifier ["as" name] )*\n | "from" relative_module "import" "(" identifier ["as" name]\n ( "," identifier ["as" name] )* [","] ")"\n | "from" module "import" "*"\n module ::= (identifier ".")* identifier\n relative_module ::= "."* module | "."+\n name ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns ``None`` is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any. If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound. As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname". If a name is not\nfound, ``ImportError`` is raised. If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement..\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module. The names given in ``__all__`` are all considered public\nand are required to exist. If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope. If the\nwild card form of import --- ``import *`` --- is used in a function\nand the function contains or is a nested block with free variables,\nthe compiler will raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimport mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python. The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language. It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n future_statement ::= "from" "__future__" "import" feature ["as" name]\n ("," feature ["as" name])*\n | "from" "__future__" "import" "(" feature ["as" name]\n ("," feature ["as" name])* [","] ")"\n feature ::= identifier\n name ::= identifier\n\nA future statement must appear near the top of the module. The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 2.6 are ``unicode_literals``,\n``print_function``, ``absolute_import``, ``division``, ``generators``,\n``nested_scopes`` and ``with_statement``. ``generators``,\n``with_statement``, ``nested_scopes`` are redundant in Python version\n2.6 and above because they are always enabled.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code. It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently. Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by an ``exec`` statement or calls to the built-in\nfunctions ``compile()`` and ``execfile()`` that occur in a module\n``M`` containing a future statement will, by default, use the new\nsyntax or semantics associated with the future statement. This can,\nstarting with Python 2.2 be controlled by optional arguments to\n``compile()`` --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session. If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n **PEP 236** - Back to the __future__\n The original proposal for the __future__ mechanism.\n', 'in': '\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation. Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n comparison ::= or_expr ( comp_operator or_expr )*\n comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "<>" | "!="\n | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe forms ``<>`` and ``!=`` are equivalent; for consistency with C,\n``!=`` is preferred; where ``!=`` is mentioned below ``<>`` is also\naccepted. The ``<>`` spelling is considered obsolescent.\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects. The objects need not have the same type.\nIf both are numbers, they are converted to a common type. Otherwise,\nobjects of different types *always* compare unequal, and are ordered\nconsistently but arbitrarily. You can control comparison behavior of\nobjects of non-built-in types by defining a ``__cmp__`` method or rich\ncomparison methods like ``__gt__``, described in section *Special\nmethod names*.\n\n(This unusual definition of comparison was used to simplify the\ndefinition of operations like sorting and the ``in`` and ``not in``\noperators. In the future, the comparison rules for objects of\ndifferent types are likely to change.)\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* Strings are compared lexicographically using the numeric equivalents\n (the result of the built-in function ``ord()``) of their characters.\n Unicode and 8-bit strings are fully interoperable in this behavior.\n [4]\n\n* Tuples and lists are compared lexicographically using comparison of\n corresponding elements. This means that to compare equal, each\n element must compare equal and the two sequences must be of the same\n type and have the same length.\n\n If not equal, the sequences are ordered the same as their first\n differing elements. For example, ``cmp([1,2,x], [1,2,y])`` returns\n the same as ``cmp(x,y)``. If the corresponding element does not\n exist, the shorter sequence is ordered first (for example, ``[1,2] <\n [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if their sorted\n (key, value) lists compare equal. [5] Outcomes other than equality\n are resolved consistently, but are not otherwise defined. [6]\n\n* Most other objects of built-in types compare unequal unless they are\n the same object; the choice whether one object is considered smaller\n or larger than another one is made arbitrarily but consistently\n within one execution of a program.\n\nThe operators ``in`` and ``not in`` test for collection membership.\n``x in s`` evaluates to true if *x* is a member of the collection *s*,\nand false otherwise. ``x not in s`` returns the negation of ``x in\ns``. The collection membership test has traditionally been bound to\nsequences; an object is a member of a collection if the collection is\na sequence and contains an element equal to that object. However, it\nmake sense for many other object types to support membership tests\nwithout being a sequence. In particular, dictionaries (for keys) and\nsets support membership testing.\n\nFor the list and tuple types, ``x in y`` is true if and only if there\nexists an index *i* such that ``x == y[i]`` is true.\n\nFor the Unicode and string types, ``x in y`` is true if and only if\n*x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``.\nNote, *x* and *y* need not be the same type; consequently, ``u\'ab\' in\n\'abc\'`` will return ``True``. Empty strings are always considered to\nbe a substring of any other string, so ``"" in "abc"`` will return\n``True``.\n\nChanged in version 2.3: Previously, *x* was required to be a string of\nlength ``1``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``. If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception. (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object. ``x is\nnot y`` yields the inverse truth value. [7]\n', 'integers': '\nInteger and long integer literals\n*********************************\n\nInteger and long integer literals are described by the following\nlexical definitions:\n\n longinteger ::= integer ("l" | "L")\n integer ::= decimalinteger | octinteger | hexinteger | bininteger\n decimalinteger ::= nonzerodigit digit* | "0"\n octinteger ::= "0" ("o" | "O") octdigit+ | "0" octdigit+\n hexinteger ::= "0" ("x" | "X") hexdigit+\n bininteger ::= "0" ("b" | "B") bindigit+\n nonzerodigit ::= "1"..."9"\n octdigit ::= "0"..."7"\n bindigit ::= "0" | "1"\n hexdigit ::= digit | "a"..."f" | "A"..."F"\n\nAlthough both lower case ``\'l\'`` and upper case ``\'L\'`` are allowed as\nsuffix for long integers, it is strongly recommended to always use\n``\'L\'``, since the letter ``\'l\'`` looks too much like the digit\n``\'1\'``.\n\nPlain integer literals that are above the largest representable plain\ninteger (e.g., 2147483647 when using 32-bit arithmetic) are accepted\nas if they were long integers instead. [1] There is no limit for long\ninteger literals apart from what can be stored in available memory.\n\nSome examples of plain integer literals (first row) and long integer\nliterals (second and third rows):\n\n 7 2147483647 0177\n 3L 79228162514264337593543950336L 0377L 0x100000000L\n 79228162514264337593543950336 0xdeadbeef\n', 'lambda': '\nLambdas\n*******\n\n lambda_form ::= "lambda" [parameter_list]: expression\n old_lambda_form ::= "lambda" [parameter_list]: old_expression\n\nLambda forms (lambda expressions) have the same syntactic position as\nexpressions. They are a shorthand to create anonymous functions; the\nexpression ``lambda arguments: expression`` yields a function object.\nThe unnamed object behaves like a function object defined with\n\n def name(arguments):\n return expression\n\nSee section *Function definitions* for the syntax of parameter lists.\nNote that functions created with lambda forms cannot contain\nstatements.\n', @@ -49,7 +49,7 @@ 'numbers': "\nNumeric literals\n****************\n\nThere are four types of numeric literals: plain integers, long\nintegers, floating point numbers, and imaginary numbers. There are no\ncomplex literals (complex numbers can be formed by adding a real\nnumber and an imaginary number).\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator '``-``' and\nthe literal ``1``.\n", 'numeric-types': '\nEmulating numeric types\n***********************\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n', 'objects': '\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data. All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value. An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory. The \'``is``\' operator compares the\nidentity of two objects; the ``id()`` function returns an integer\nrepresenting its identity (currently implemented as its address). An\nobject\'s *type* is also unchangeable. [1] An object\'s type determines\nthe operations that the object supports (e.g., "does it have a\nlength?") and also defines the possible values for objects of that\ntype. The ``type()`` function returns an object\'s type (which is an\nobject itself). The *value* of some objects can change. Objects\nwhose value can change are said to be *mutable*; objects whose value\nis unchangeable once they are created are called *immutable*. (The\nvalue of an immutable container object that contains a reference to a\nmutable object can change when the latter\'s value is changed; however\nthe container is still considered immutable, because the collection of\nobjects it contains cannot be changed. So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected. An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references. See the documentation of the ``gc`` module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change. Do not depend\non immediate finalization of objects when they become unreachable (ex:\nalways close files).\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'``try``...``except``\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows. It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a ``close()`` method. Programs\nare strongly recommended to explicitly close such objects. The\n\'``try``...``finally``\' statement provides a convenient way to do\nthis.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries. The references are part of a container\'s value. In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied. So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior. Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed. E.g., after ``a = 1; b =\n1``, ``a`` and ``b`` may or may not refer to the same object with the\nvalue one, depending on the implementation, but after ``c = []; d =\n[]``, ``c`` and ``d`` are guaranteed to refer to two different,\nunique, newly created empty lists. (Note that ``c = d = []`` assigns\nthe same object to both ``c`` and ``d``.)\n', - 'operator-summary': '\nOperator precedence\n*******************\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| ``lambda`` | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else`` | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| ``or`` | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| ``and`` | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` ``x`` | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not in``, ``is``, ``is not``, ``<``, | Comparisons, including membership |\n| ``<=``, ``>``, ``>=``, ``<>``, ``!=``, ``==`` | tests and identity tests, |\n+-------------------------------------------------+---------------------------------------+\n| ``|`` | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| ``^`` | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| ``&`` | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>`` | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-`` | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder |\n| | [8] |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``**`` | Exponentiation [9] |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``, | Subscription, slicing, call, |\n| ``x(arguments...)``, ``x.attribute`` | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``, | Binding or tuple display, list |\n| ``{key: value...}``, ```expressions...``` | display, dictionary display, string |\n| | conversion |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n control variables of each ``for`` it contains into the containing\n scope. However, this behavior is deprecated, and relying on it\n will not work in Python 3.\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that ``-1e-100 % 1e100`` have the same\n sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n which is numerically exactly equal to ``1e100``. The function\n ``math.fmod()`` returns a result whose sign matches the sign of\n the first argument instead, and so returns ``-1e-100`` in this\n case. Which approach is more appropriate depends on the\n application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n due to rounding. In such cases, Python returns the latter result,\n in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n even though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n sorted (key, value) lists, but this was very expensive for the\n common case of comparing for equality. An even earlier version of\n Python compared dictionaries by identity only, but this caused\n surprises because people expected to be able to test a dictionary\n for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n nature of descriptors, you may notice seemingly unusual behaviour\n in certain uses of the ``is`` operator, like those involving\n comparisons between instance methods, or constants. Check their\n documentation for more info.\n\n[8] The ``%`` operator is also used for string formatting; the same\n precedence applies.\n\n[9] The power operator ``**`` binds less tightly than an arithmetic or\n bitwise unary operator on its right, that is, ``2**-1`` is\n ``0.5``.\n', + 'operator-summary': '\nOperator precedence\n*******************\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| ``lambda`` | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else`` | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| ``or`` | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| ``and`` | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` ``x`` | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not in``, ``is``, ``is not``, ``<``, | Comparisons, including membership |\n| ``<=``, ``>``, ``>=``, ``<>``, ``!=``, ``==`` | tests and identity tests |\n+-------------------------------------------------+---------------------------------------+\n| ``|`` | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| ``^`` | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| ``&`` | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>`` | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-`` | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder |\n| | [8] |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``**`` | Exponentiation [9] |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``, | Subscription, slicing, call, |\n| ``x(arguments...)``, ``x.attribute`` | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``, | Binding or tuple display, list |\n| ``{key: value...}``, ```expressions...``` | display, dictionary display, string |\n| | conversion |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n control variables of each ``for`` it contains into the containing\n scope. However, this behavior is deprecated, and relying on it\n will not work in Python 3.\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that ``-1e-100 % 1e100`` have the same\n sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n which is numerically exactly equal to ``1e100``. The function\n ``math.fmod()`` returns a result whose sign matches the sign of\n the first argument instead, and so returns ``-1e-100`` in this\n case. Which approach is more appropriate depends on the\n application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n due to rounding. In such cases, Python returns the latter result,\n in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n even though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n sorted (key, value) lists, but this was very expensive for the\n common case of comparing for equality. An even earlier version of\n Python compared dictionaries by identity only, but this caused\n surprises because people expected to be able to test a dictionary\n for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n nature of descriptors, you may notice seemingly unusual behaviour\n in certain uses of the ``is`` operator, like those involving\n comparisons between instance methods, or constants. Check their\n documentation for more info.\n\n[8] The ``%`` operator is also used for string formatting; the same\n precedence applies.\n\n[9] The power operator ``**`` binds less tightly than an arithmetic or\n bitwise unary operator on its right, that is, ``2**-1`` is\n ``0.5``.\n', 'pass': '\nThe ``pass`` statement\n**********************\n\n pass_stmt ::= "pass"\n\n``pass`` is a null operation --- when it is executed, nothing happens.\nIt is useful as a placeholder when a statement is required\nsyntactically, but no code needs to be executed, for example:\n\n def f(arg): pass # a function that does nothing (yet)\n\n class C: pass # a class with no methods (yet)\n', 'power': '\nThe power operator\n******************\n\nThe power operator binds more tightly than unary operators on its\nleft; it binds less tightly than unary operators on its right. The\nsyntax is:\n\n power ::= primary ["**" u_expr]\n\nThus, in an unparenthesized sequence of power and unary operators, the\noperators are evaluated from right to left (this does not constrain\nthe evaluation order for the operands): ``-1**2`` results in ``-1``.\n\nThe power operator has the same semantics as the built-in ``pow()``\nfunction, when called with two arguments: it yields its left argument\nraised to the power of its right argument. The numeric arguments are\nfirst converted to a common type. The result type is that of the\narguments after coercion.\n\nWith mixed operand types, the coercion rules for binary arithmetic\noperators apply. For int and long int operands, the result has the\nsame type as the operands (after coercion) unless the second argument\nis negative; in that case, all arguments are converted to float and a\nfloat result is delivered. For example, ``10**2`` returns ``100``, but\n``10**-2`` returns ``0.01``. (This last feature was added in Python\n2.2. In Python 2.1 and before, if both arguments were of integer types\nand the second argument was negative, an exception was raised).\n\nRaising ``0.0`` to a negative power results in a\n``ZeroDivisionError``. Raising a negative number to a fractional power\nresults in a ``ValueError``.\n', 'raise': '\nThe ``raise`` statement\n***********************\n\n raise_stmt ::= "raise" [expression ["," expression ["," expression]]]\n\nIf no expressions are present, ``raise`` re-raises the last exception\nthat was active in the current scope. If no exception is active in\nthe current scope, a ``TypeError`` exception is raised indicating that\nthis is an error (if running under IDLE, a ``Queue.Empty`` exception\nis raised instead).\n\nOtherwise, ``raise`` evaluates the expressions to get three objects,\nusing ``None`` as the value of omitted expressions. The first two\nobjects are used to determine the *type* and *value* of the exception.\n\nIf the first object is an instance, the type of the exception is the\nclass of the instance, the instance itself is the value, and the\nsecond object must be ``None``.\n\nIf the first object is a class, it becomes the type of the exception.\nThe second object is used to determine the exception value: If it is\nan instance of the class, the instance becomes the exception value. If\nthe second object is a tuple, it is used as the argument list for the\nclass constructor; if it is ``None``, an empty argument list is used,\nand any other object is treated as a single argument to the\nconstructor. The instance so created by calling the constructor is\nused as the exception value.\n\nIf a third object is present and not ``None``, it must be a traceback\nobject (see section *The standard type hierarchy*), and it is\nsubstituted instead of the current location as the place where the\nexception occurred. If the third object is present and not a\ntraceback object or ``None``, a ``TypeError`` exception is raised.\nThe three-expression form of ``raise`` is useful to re-raise an\nexception transparently in an except clause, but ``raise`` with no\nexpressions should be preferred if the exception to be re-raised was\nthe most recently active exception in the current scope.\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information about handling exceptions is in section\n*The try statement*.\n', @@ -59,19 +59,19 @@ 'slicings': '\nSlicings\n********\n\nA slicing selects a range of items in a sequence object (e.g., a\nstring, tuple or list). Slicings may be used as expressions or as\ntargets in assignment or ``del`` statements. The syntax for a\nslicing:\n\n slicing ::= simple_slicing | extended_slicing\n simple_slicing ::= primary "[" short_slice "]"\n extended_slicing ::= primary "[" slice_list "]"\n slice_list ::= slice_item ("," slice_item)* [","]\n slice_item ::= expression | proper_slice | ellipsis\n proper_slice ::= short_slice | long_slice\n short_slice ::= [lower_bound] ":" [upper_bound]\n long_slice ::= short_slice ":" [stride]\n lower_bound ::= expression\n upper_bound ::= expression\n stride ::= expression\n ellipsis ::= "..."\n\nThere is ambiguity in the formal syntax here: anything that looks like\nan expression list also looks like a slice list, so any subscription\ncan be interpreted as a slicing. Rather than further complicating the\nsyntax, this is disambiguated by defining that in this case the\ninterpretation as a subscription takes priority over the\ninterpretation as a slicing (this is the case if the slice list\ncontains no proper slice nor ellipses). Similarly, when the slice\nlist has exactly one short slice and no trailing comma, the\ninterpretation as a simple slicing takes priority over that as an\nextended slicing.\n\nThe semantics for a simple slicing are as follows. The primary must\nevaluate to a sequence object. The lower and upper bound expressions,\nif present, must evaluate to plain integers; defaults are zero and the\n``sys.maxint``, respectively. If either bound is negative, the\nsequence\'s length is added to it. The slicing now selects all items\nwith index *k* such that ``i <= k < j`` where *i* and *j* are the\nspecified lower and upper bounds. This may be an empty sequence. It\nis not an error if *i* or *j* lie outside the range of valid indexes\n(such items don\'t exist so they aren\'t selected).\n\nThe semantics for an extended slicing are as follows. The primary\nmust evaluate to a mapping object, and it is indexed with a key that\nis constructed from the slice list, as follows. If the slice list\ncontains at least one comma, the key is a tuple containing the\nconversion of the slice items; otherwise, the conversion of the lone\nslice item is the key. The conversion of a slice item that is an\nexpression is that expression. The conversion of an ellipsis slice\nitem is the built-in ``Ellipsis`` object. The conversion of a proper\nslice is a slice object (see section *The standard type hierarchy*)\nwhose ``start``, ``stop`` and ``step`` attributes are the values of\nthe expressions given as lower bound, upper bound and stride,\nrespectively, substituting ``None`` for missing expressions.\n', 'specialattrs': '\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant. Some of these are not reported\nby the ``dir()`` built-in function.\n\nobject.__dict__\n\n A dictionary or other mapping object used to store an object\'s\n (writable) attributes.\n\nobject.__methods__\n\n Deprecated since version 2.2: Use the built-in function ``dir()``\n to get a list of an object\'s attributes. This attribute is no\n longer available.\n\nobject.__members__\n\n Deprecated since version 2.2: Use the built-in function ``dir()``\n to get a list of an object\'s attributes. This attribute is no\n longer available.\n\ninstance.__class__\n\n The class to which a class instance belongs.\n\nclass.__bases__\n\n The tuple of base classes of a class object.\n\nclass.__name__\n\n The name of the class or type.\n\nThe following attributes are only supported by *new-style class*es.\n\nclass.__mro__\n\n This attribute is a tuple of classes that are considered when\n looking for base classes during method resolution.\n\nclass.mro()\n\n This method can be overridden by a metaclass to customize the\n method resolution order for its instances. It is called at class\n instantiation, and its result is stored in ``__mro__``.\n\nclass.__subclasses__()\n\n Each new-style class keeps a list of weak references to its\n immediate subclasses. This method returns a list of all those\n references still alive. Example:\n\n >>> int.__subclasses__()\n []\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found in\n the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list ``[1, 2]`` is considered equal to\n ``[1.0, 2.0]``, and similarly for tuples.\n\n[3] They must have since the parser can\'t tell the type of the\n operands.\n\n[4] Cased characters are those with general category property being\n one of "Lu" (Letter, uppercase), "Ll" (Letter, lowercase), or "Lt"\n (Letter, titlecase).\n\n[5] To format only a tuple you should therefore provide a singleton\n tuple whose only element is the tuple to be formatted.\n\n[6] The advantage of leaving the newline on is that returning an empty\n string is then an unambiguous EOF indication. It is also possible\n (in cases where it might matter, for example, if you want to make\n an exact copy of a file while scanning its lines) to tell whether\n the last line of a file ended in a newline or not (yes this\n happens!).\n', 'specialnames': '\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``x.__getitem__(i)`` for\nold-style classes and ``type(x).__getitem__(x, i)`` for new-style\nclasses. Except where mentioned, attempts to execute an operation\nraise an exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.exc_traceback`` or ``sys.last_traceback``. Circular\n references which are garbage are detected when the option cycle\n detector is enabled (it\'s on by default), but can only be cleaned\n up if there are no Python-level ``__del__()`` methods involved.\n Refer to the documentation for the ``gc`` module for more\n information about how ``__del__()`` methods are handled by the\n cycle detector, particularly the description of the ``garbage``\n value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\n See also the *-R* command-line option.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function and by string\n conversions (reverse quotes) to compute the "official" string\n representation of an object. If at all possible, this should look\n like a valid Python expression that could be used to recreate an\n object with the same value (given an appropriate environment). If\n this is not possible, a string of the form ``<...some useful\n description...>`` should be returned. The return value must be a\n string object. If a class defines ``__repr__()`` but not\n ``__str__()``, then ``__repr__()`` is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print``\n statement to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n New in version 2.1.\n\n These are the so-called "rich comparison" methods, and are called\n for comparison operators in preference to ``__cmp__()`` below. The\n correspondence between operator symbols and method names is as\n follows: ``xy`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n ``x>=y`` calls ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n Called by comparison operations if rich comparison (see above) is\n not defined. Should return a negative integer if ``self < other``,\n zero if ``self == other``, a positive integer if ``self > other``.\n If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n defined, class instances are compared by object identity\n ("address"). See also the description of ``__hash__()`` for some\n important notes on creating *hashable* objects which support custom\n comparison operations and are usable as dictionary keys. (Note: the\n restriction that exceptions are not propagated by ``__cmp__()`` has\n been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n it should not define a ``__hash__()`` operation either; if it\n defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n instances will not be usable in hashed collections. If a class\n defines mutable objects and implements a ``__cmp__()`` or\n ``__eq__()`` method, it should not implement ``__hash__()``, since\n hashable collection implementations require that a object\'s hash\n value is immutable (if the object\'s hash value changes, it will be\n in the wrong hash bucket).\n\n User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n the hash value returned is no longer appropriate (e.g. by switching\n to a value-based concept of equality instead of the default\n identity based equality) can explicitly flag themselves as being\n unhashable by setting ``__hash__ = None`` in the class definition.\n Doing so means that not only will instances of the class raise an\n appropriate ``TypeError`` when a program attempts to retrieve their\n hash value, but they will also be correctly identified as\n unhashable when checking ``isinstance(obj, collections.Hashable)``\n (unlike classes which define their own ``__hash__()`` to explicitly\n raise ``TypeError``).\n\n Changed in version 2.5: ``__hash__()`` may now also return a long\n integer object; the 32-bit integer is then derived from the hash of\n that object.\n\n Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``, or their integer\n equivalents ``0`` or ``1``. When this method is not defined,\n ``__len__()`` is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n considered true.\n\nobject.__unicode__(self)\n\n Called to implement ``unicode()`` built-in; should return a Unicode\n object. When this method is not defined, string conversion is\n attempted, and the result of string conversion is converted to\n Unicode using the system default encoding.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n-------------------------------------------\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, new-style classes are constructed using ``type()``. A\nclass definition is read into a separate namespace and the value of\nclass name is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if *__metaclass__* is defined then\nthe callable assigned to it will be called instead of ``type()``. This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties. This example adds a new\nelement to the class dictionary before creating the class:\n\n class metacls(type):\n def __new__(mcs, name, bases, dict):\n dict[\'foo\'] = \'metacls was here\'\n return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\n__metaclass__\n\n This variable can be any callable accepting arguments for ``name``,\n ``bases``, and ``dict``. Upon class creation, the callable is used\n instead of the built-in ``type()``.\n\n New in version 2.2.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If ``dict[\'__metaclass__\']`` exists, it is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n used (this looks for a *__class__* attribute first and if not found,\n uses its type).\n\n* Otherwise, if a global variable named __metaclass__ exists, it is\n used.\n\n* Otherwise, the old-style, classic metaclass (types.ClassType) is\n used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\n\nCustomizing instance and subclass checks\n========================================\n\nNew in version 2.6.\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n **PEP 3119** - Introducing Abstract Base Classes\n Includes the specification for customizing ``isinstance()`` and\n ``issubclass()`` behavior through ``__instancecheck__()`` and\n ``__subclasscheck__()``, with motivation for this functionality\n in the context of adding Abstract Base Classes (see the ``abc``\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. (For backwards compatibility, the method\n``__getslice__()`` (see below) can also be defined to handle simple,\nbut not extended slices.) It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``has_key()``,\n``get()``, ``clear()``, ``setdefault()``, ``iterkeys()``,\n``itervalues()``, ``iteritems()``, ``pop()``, ``popitem()``,\n``copy()``, and ``update()`` behaving similar to those for Python\'s\nstandard dictionary objects. The ``UserDict`` module provides a\n``DictMixin`` class to help create those methods from a base set of\n``__getitem__()``, ``__setitem__()``, ``__delitem__()``, and\n``keys()``. Mutable sequences should provide methods ``append()``,\n``count()``, ``index()``, ``extend()``, ``insert()``, ``pop()``,\n``remove()``, ``reverse()`` and ``sort()``, like Python standard list\nobjects. Finally, sequence types should implement addition (meaning\nconcatenation) and multiplication (meaning repetition) by defining the\nmethods ``__add__()``, ``__radd__()``, ``__iadd__()``, ``__mul__()``,\n``__rmul__()`` and ``__imul__()`` described below; they should not\ndefine ``__coerce__()`` or other numerical operators. It is\nrecommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should be equivalent of ``has_key()``;\nfor sequences, it should search through the values. It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``iterkeys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function ``len()``. Should return\n the length of the object, an integer ``>=`` 0. Also, an object\n that doesn\'t define a ``__nonzero__()`` method and whose\n ``__len__()`` method returns zero is considered to be false in a\n Boolean context.\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of ``self[key]``. For sequence\n types, the accepted keys should be integers and slice objects.\n Note that the special interpretation of negative indexes (if the\n class wishes to emulate a sequence type) is up to the\n ``__getitem__()`` method. If *key* is of an inappropriate type,\n ``TypeError`` may be raised; if of a value outside the set of\n indexes for the sequence (after any special interpretation of\n negative values), ``IndexError`` should be raised. For mapping\n types, if *key* is missing (not in the container), ``KeyError``\n should be raised.\n\n Note: ``for`` loops expect that an ``IndexError`` will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the ``__getitem__()``\n method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method ``iterkeys()``.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the ``reversed()`` built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the ``__reversed__()`` method is not provided, the\n ``reversed()`` built-in will fall back to using the sequence\n protocol (``__len__()`` and ``__getitem__()``). Objects that\n support the sequence protocol should only provide\n ``__reversed__()`` if they can provide an implementation that is\n more efficient than the one provided by ``reversed()``.\n\n New in version 2.6.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define ``__contains__()``, the membership\n test first tries iteration via ``__iter__()``, then the old\n sequence iteration protocol via ``__getitem__()``, see *this\n section in the language reference*.\n\n\nAdditional methods for emulation of sequence types\n==================================================\n\nThe following optional methods can be defined to further emulate\nsequence objects. Immutable sequences methods should at most only\ndefine ``__getslice__()``; mutable sequences might define all three\nmethods.\n\nobject.__getslice__(self, i, j)\n\n Deprecated since version 2.0: Support slice objects as parameters\n to the ``__getitem__()`` method. (However, built-in types in\n CPython currently still implement ``__getslice__()``. Therefore,\n you have to override it in derived classes when implementing\n slicing.)\n\n Called to implement evaluation of ``self[i:j]``. The returned\n object should be of the same type as *self*. Note that missing *i*\n or *j* in the slice expression are replaced by zero or\n ``sys.maxint``, respectively. If negative indexes are used in the\n slice, the length of the sequence is added to that index. If the\n instance does not implement the ``__len__()`` method, an\n ``AttributeError`` is raised. No guarantee is made that indexes\n adjusted this way are not still negative. Indexes which are\n greater than the length of the sequence are not modified. If no\n ``__getslice__()`` is found, a slice object is created instead, and\n passed to ``__getitem__()`` instead.\n\nobject.__setslice__(self, i, j, sequence)\n\n Called to implement assignment to ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``.\n\n This method is deprecated. If no ``__setslice__()`` is found, or\n for extended slicing of the form ``self[i:j:k]``, a slice object is\n created, and passed to ``__setitem__()``, instead of\n ``__setslice__()`` being called.\n\nobject.__delslice__(self, i, j)\n\n Called to implement deletion of ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``. This method is deprecated. If no\n ``__delslice__()`` is found, or for extended slicing of the form\n ``self[i:j:k]``, a slice object is created, and passed to\n ``__delitem__()``, instead of ``__delslice__()`` being called.\n\nNotice that these methods are only invoked when a single slice with a\nsingle colon is used, and the slice method is available. For slice\noperations involving extended slice notation, or in absence of the\nslice methods, ``__getitem__()``, ``__setitem__()`` or\n``__delitem__()`` is called with a slice object as argument.\n\nThe following example demonstrate how to make your program or module\ncompatible with earlier versions of Python (assuming that methods\n``__getitem__()``, ``__setitem__()`` and ``__delitem__()`` support\nslice objects as arguments):\n\n class MyClass:\n ...\n def __getitem__(self, index):\n ...\n def __setitem__(self, index, value):\n ...\n def __delitem__(self, index):\n ...\n\n if sys.version_info < (2, 0):\n # They won\'t be defined if version is at least 2.0 final\n\n def __getslice__(self, i, j):\n return self[max(0, i):max(0, j):]\n def __setslice__(self, i, j, seq):\n self[max(0, i):max(0, j):] = seq\n def __delslice__(self, i, j):\n del self[max(0, i):max(0, j):]\n ...\n\nNote the calls to ``max()``; these are necessary because of the\nhandling of negative indices before the ``__*slice__()`` methods are\ncalled. When negative indexes are used, the ``__*item__()`` methods\nreceive them as provided, but the ``__*slice__()`` methods get a\n"cooked" form of the index values. For each negative index value, the\nlength of the sequence is added to the index before calling the method\n(which may still result in a negative index); this is the customary\nhandling of negative indexes by the built-in sequence types, and the\n``__*item__()`` methods are expected to do this as well. However,\nsince they should already be doing that, negative indexes cannot be\npassed in; they must be constrained to the bounds of the sequence\nbefore being passed to the ``__*item__()`` methods. Calling ``max(0,\ni)`` conveniently returns the proper value.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n\n\nCoercion rules\n==============\n\nThis section used to document the rules for coercion. As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable. Instead, here are some informal\nguidelines regarding coercion. In Python 3, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n no coercion takes place and the string formatting operation is\n invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n mode operations on types that don\'t define coercion pass the\n original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n ``__coerce__()`` method in response to a binary operator; the only\n time ``__coerce__()`` is invoked is when the built-in function\n ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n ``NotImplemented`` is treated the same as one that is not\n implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n generic method names corresponding to an operator; ``__iop__()`` is\n used for the corresponding in-place operator. For example, for the\n operator \'``+``\', ``__add__()`` and ``__radd__()`` are used for the\n left and right variant of the binary operator, and ``__iadd__()``\n for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried. If this is\n not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n tried. If this is also not implemented or returns\n ``NotImplemented``, a ``TypeError`` exception is raised. But see\n the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n of a built-in type or a new-style class, and the right operand is an\n instance of a proper subclass of that type or class and overrides\n the base\'s ``__rop__()`` method, the right operand\'s ``__rop__()``\n method is tried *before* the left operand\'s ``__op__()`` method.\n\n This is done so that a subclass can completely override binary\n operators. Otherwise, the left operand\'s ``__op__()`` method would\n always accept the right operand: when an instance of a given class\n is expected, an instance of a subclass of that class is always\n acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n before that type\'s ``__op__()`` or ``__rop__()`` method is called,\n but no sooner. If the coercion returns an object of a different\n type for the operand whose coercion is invoked, part of the process\n is redone using the new object.\n\n* When an in-place operator (like \'``+=``\') is used, if the left\n operand implements ``__iop__()``, it is invoked without any\n coercion. When the operation falls back to ``__op__()`` and/or\n ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operand is a sequence that implements sequence\n repetition, and the other is an integer (``int`` or ``long``),\n sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n never use coercion. Three-way comparison (implemented by\n ``__cmp__()``) does use coercion under the same conditions as other\n binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n ``long``, ``float``, and ``complex`` do not use coercion. All these\n types implement a ``__coerce__()`` method, for use by the built-in\n ``coerce()`` function.\n\n Changed in version 2.7.\n\n\nWith Statement Context Managers\n===============================\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nSpecial method lookup for old-style classes\n===========================================\n\nFor old-style classes, special methods are always looked up in exactly\nthe same way as any other method or attribute. This is the case\nregardless of whether the method is being looked up explicitly as in\n``x.__getitem__(i)`` or implicitly as in ``x[i]``.\n\nThis behaviour means that special methods may exhibit different\nbehaviour for different instances of a single old-style class if the\nappropriate special attributes are set differently:\n\n >>> class C:\n ... pass\n ...\n >>> c1 = C()\n >>> c2 = C()\n >>> c1.__len__ = lambda: 5\n >>> c2.__len__ = lambda: 9\n >>> len(c1)\n 5\n >>> len(c2)\n 9\n\n\nSpecial method lookup for new-style classes\n===========================================\n\nFor new-style classes, implicit invocations of special methods are\nonly guaranteed to work correctly if defined on an object\'s type, not\nin the object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception (unlike the equivalent example\nwith old-style classes):\n\n >>> class C(object):\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "", line 1, in \n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "", line 1, in \n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print "Metaclass getattribute invoked"\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object):\n ... __metaclass__ = Meta\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print "Class getattribute invoked"\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n certain controlled conditions. It generally isn\'t a good idea\n though, since it can lead to some very strange behaviour if it is\n handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n reflected method (such as ``__add__()``) fails the operation is\n not supported, which is why the reflected method is not called.\n', - 'string-methods': '\nString Methods\n**************\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters [4] in the string are lowercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters [4] in the string are uppercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string with all the cased characters [4]\n converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified or ``-1``, then there is\n no limit on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. This method uses the *universal newlines* approach to\n splitting lines. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\n For example, ``\'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()`` returns\n ``[\'ab c\', \'\', \'de fg\', \'kl\']``, while the same call with\n ``splitlines(True)`` returns ``[\'ab c\\n\', \'\\n\', \'de fg\\r\',\n \'kl\\r\\n\']``.\n\n Unlike ``split()`` when a delimiter string *sep* is given, this\n method returns an empty list for the empty string, and a terminal\n line break does not result in an extra line.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n ... lambda mo: mo.group(0)[0].upper() +\n ... mo.group(0)[1:].lower(),\n ... s)\n ...\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string with all the cased characters [4]\n converted to uppercase. Note that ``str.upper().isupper()`` might\n be ``False`` if ``s`` contains uncased characters or if the Unicode\n category of the resulting character(s) is not "Lu" (Letter,\n uppercase), but e.g. "Lt" (Letter, titlecase).\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than or equal to ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that can be used to form decimal-radix numbers,\n e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n', + 'string-methods': '\nString Methods\n**************\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. Tab positions occur every *tabsize* characters\n (default is 8, giving tab positions at columns 0, 8, 16 and so on).\n To expand the string, the current column is set to zero and the\n string is examined character by character. If the character is a\n tab (``\\t``), one or more space characters are inserted in the\n result until the current column is equal to the next tab position.\n (The tab character itself is not copied.) If the character is a\n newline (``\\n``) or return (``\\r``), it is copied and the current\n column is reset to zero. Any other character is copied unchanged\n and the current column is incremented by one regardless of how the\n character is represented when printed.\n\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs()\n \'01 012 0123 01234\'\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs(4)\n \'01 012 0123 01234\'\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters [4] in the string are lowercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters [4] in the string are uppercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string with all the cased characters [4]\n converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified or ``-1``, then there is\n no limit on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. This method uses the *universal newlines* approach to\n splitting lines. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\n For example, ``\'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()`` returns\n ``[\'ab c\', \'\', \'de fg\', \'kl\']``, while the same call with\n ``splitlines(True)`` returns ``[\'ab c\\n\', \'\\n\', \'de fg\\r\',\n \'kl\\r\\n\']``.\n\n Unlike ``split()`` when a delimiter string *sep* is given, this\n method returns an empty list for the empty string, and a terminal\n line break does not result in an extra line.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n ... lambda mo: mo.group(0)[0].upper() +\n ... mo.group(0)[1:].lower(),\n ... s)\n ...\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string with all the cased characters [4]\n converted to uppercase. Note that ``str.upper().isupper()`` might\n be ``False`` if ``s`` contains uncased characters or if the Unicode\n category of the resulting character(s) is not "Lu" (Letter,\n uppercase), but e.g. "Lt" (Letter, titlecase).\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than or equal to ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that can be used to form decimal-radix numbers,\n e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n', 'strings': '\nString literals\n***************\n\nString literals are described by the following lexical definitions:\n\n stringliteral ::= [stringprefix](shortstring | longstring)\n stringprefix ::= "r" | "u" | "ur" | "R" | "U" | "UR" | "Ur" | "uR"\n | "b" | "B" | "br" | "Br" | "bR" | "BR"\n shortstring ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n longstring ::= "\'\'\'" longstringitem* "\'\'\'"\n | \'"""\' longstringitem* \'"""\'\n shortstringitem ::= shortstringchar | escapeseq\n longstringitem ::= longstringchar | escapeseq\n shortstringchar ::= \n longstringchar ::= \n escapeseq ::= "\\" \n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the ``stringprefix`` and the rest of\nthe string literal. The source character set is defined by the\nencoding declaration; it is ASCII if no encoding declaration is given\nin the source file; see section *Encoding declarations*.\n\nIn plain English: String literals can be enclosed in matching single\nquotes (``\'``) or double quotes (``"``). They can also be enclosed in\nmatching groups of three single or double quotes (these are generally\nreferred to as *triple-quoted strings*). The backslash (``\\``)\ncharacter is used to escape characters that otherwise have a special\nmeaning, such as newline, backslash itself, or the quote character.\nString literals may optionally be prefixed with a letter ``\'r\'`` or\n``\'R\'``; such strings are called *raw strings* and use different rules\nfor interpreting backslash escape sequences. A prefix of ``\'u\'`` or\n``\'U\'`` makes the string a Unicode string. Unicode strings use the\nUnicode character set as defined by the Unicode Consortium and ISO\n10646. Some additional escape sequences, described below, are\navailable in Unicode strings. A prefix of ``\'b\'`` or ``\'B\'`` is\nignored in Python 2; it indicates that the literal should become a\nbytes literal in Python 3 (e.g. when code is automatically converted\nwith 2to3). A ``\'u\'`` or ``\'b\'`` prefix may be followed by an ``\'r\'``\nprefix.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string. (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C. The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| ``\\newline`` | Ignored | |\n+-------------------+-----------------------------------+---------+\n| ``\\\\`` | Backslash (``\\``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\\'`` | Single quote (``\'``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\"`` | Double quote (``"``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\a`` | ASCII Bell (BEL) | |\n+-------------------+-----------------------------------+---------+\n| ``\\b`` | ASCII Backspace (BS) | |\n+-------------------+-----------------------------------+---------+\n| ``\\f`` | ASCII Formfeed (FF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\n`` | ASCII Linefeed (LF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\N{name}`` | Character named *name* in the | |\n| | Unicode database (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\r`` | ASCII Carriage Return (CR) | |\n+-------------------+-----------------------------------+---------+\n| ``\\t`` | ASCII Horizontal Tab (TAB) | |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx`` | Character with 16-bit hex value | (1) |\n| | *xxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx`` | Character with 32-bit hex value | (2) |\n| | *xxxxxxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\v`` | ASCII Vertical Tab (VT) | |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo`` | Character with octal value *ooo* | (3,5) |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh`` | Character with hex value *hh* | (4,5) |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. Individual code units which form parts of a surrogate pair can be\n encoded using this escape sequence.\n\n2. Any Unicode character can be encoded this way, but characters\n outside the Basic Multilingual Plane (BMP) will be encoded using a\n surrogate pair if Python is compiled to use 16-bit code units (the\n default). Individual code units which form parts of a surrogate\n pair can be encoded using this escape sequence.\n\n3. As in Standard C, up to three octal digits are accepted.\n\n4. Unlike in Standard C, exactly two hex digits are required.\n\n5. In a string literal, hexadecimal and octal escapes denote the byte\n with the given value; it is not necessary that the byte encodes a\n character in the source character set. In a Unicode literal, these\n escapes denote a Unicode character with the given value.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*. (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.) It is also\nimportant to note that the escape sequences marked as "(Unicode only)"\nin the table above fall into the category of unrecognized escapes for\nnon-Unicode string literals.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is present, a character following a\nbackslash is included in the string without change, and *all\nbackslashes are left in the string*. For example, the string literal\n``r"\\n"`` consists of two characters: a backslash and a lowercase\n``\'n\'``. String quotes can be escaped with a backslash, but the\nbackslash remains in the string; for example, ``r"\\""`` is a valid\nstring literal consisting of two characters: a backslash and a double\nquote; ``r"\\"`` is not a valid string literal (even a raw string\ncannot end in an odd number of backslashes). Specifically, *a raw\nstring cannot end in a single backslash* (since the backslash would\nescape the following quote character). Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is used in conjunction with a\n``\'u\'`` or ``\'U\'`` prefix, then the ``\\uXXXX`` and ``\\UXXXXXXXX``\nescape sequences are processed while *all other backslashes are left\nin the string*. For example, the string literal ``ur"\\u0062\\n"``\nconsists of three Unicode characters: \'LATIN SMALL LETTER B\', \'REVERSE\nSOLIDUS\', and \'LATIN SMALL LETTER N\'. Backslashes can be escaped with\na preceding backslash; however, both remain in the string. As a\nresult, ``\\uXXXX`` escape sequences are only recognized when there are\nan odd number of backslashes.\n', 'subscriptions': '\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object of a sequence or mapping type.\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey. (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to a\nplain integer. If this value is negative, the length of the sequence\nis added to it (so that, e.g., ``x[-1]`` selects the last item of\n``x``.) The resulting value must be a nonnegative integer less than\nthe number of items in the sequence, and the subscription selects the\nitem whose index is that value (counting from zero).\n\nA string\'s items are characters. A character is not a separate data\ntype but a string of exactly one character.\n', 'truth': "\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an ``if`` or\n``while`` condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* ``None``\n\n* ``False``\n\n* zero of any numeric type, for example, ``0``, ``0L``, ``0.0``,\n ``0j``.\n\n* any empty sequence, for example, ``''``, ``()``, ``[]``.\n\n* any empty mapping, for example, ``{}``.\n\n* instances of user-defined classes, if the class defines a\n ``__nonzero__()`` or ``__len__()`` method, when that method returns\n the integer zero or ``bool`` value ``False``. [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn ``0`` or ``False`` for false and ``1`` or ``True`` for true,\nunless otherwise stated. (Important exception: the Boolean operations\n``or`` and ``and`` always return one of their operands.)\n", 'try': '\nThe ``try`` statement\n*********************\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression [("as" | ",") target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, or a tuple containing an item compatible with the\nexception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed. All except clauses must have an\nexecutable block. When the end of this block is reached, execution\ncontinues normally after the entire try statement. (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``. Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program. As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is discarded:\n\n def f():\n try:\n 1/0\n finally:\n return 42\n\n >>> f()\n 42\n\nThe exception information is not available to the program during\nexecution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n', - 'types': '\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.).\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name ``None``.\n It is used to signify the absence of a value in many situations,\n e.g., it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``NotImplemented``. Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``Ellipsis``. It is used to indicate the presence of the ``...``\n syntax in a slice. Its truth value is true.\n\n``numbers.Number``\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n ``numbers.Integral``\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are three types of integers:\n\n Plain integers\n These represent numbers in the range -2147483648 through\n 2147483647. (The range may be larger on machines with a\n larger natural word size, but not smaller.) When the result\n of an operation would fall outside this range, the result is\n normally returned as a long integer (in some cases, the\n exception ``OverflowError`` is raised instead). For the\n purpose of shift and mask operations, integers are assumed to\n have a binary, 2\'s complement notation using 32 or more bits,\n and hiding no bits from the user (i.e., all 4294967296\n different bit patterns correspond to different values).\n\n Long integers\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans\n These represent the truth values False and True. The two\n objects representing the values False and True are the only\n Boolean objects. The Boolean type is a subtype of plain\n integers, and Boolean values behave like the values 0 and 1,\n respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ``"False"`` or\n ``"True"`` are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers and the least surprises when\n switching between the plain and long integer domains. Any\n operation, if it yields a result in the plain integer domain,\n will yield the same result in the long integer domain or when\n using mixed operands. The switch between domains is transparent\n to the programmer.\n\n ``numbers.Real`` (``float``)\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n ``numbers.Complex``\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number ``z`` can be retrieved through the read-only\n attributes ``z.real`` and ``z.imag``.\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function ``len()`` returns the number of\n items of a sequence. When the length of a sequence is *n*, the\n index set contains the numbers 0, 1, ..., *n*-1. Item *i* of\n sequence *a* is selected by ``a[i]``.\n\n Sequences also support slicing: ``a[i:j]`` selects all items with\n index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n The items of a string are characters. There is no separate\n character type; a character is represented by a string of one\n item. Characters represent (at least) 8-bit bytes. The\n built-in functions ``chr()`` and ``ord()`` convert between\n characters and nonnegative integers representing the byte\n values. Bytes with the values 0-127 usually represent the\n corresponding ASCII values, but the interpretation of values\n is up to the program. The string data type is also used to\n represent arrays of bytes, e.g., to hold data read from a\n file.\n\n (On systems whose native character set is not ASCII, strings\n may use EBCDIC in their internal representation, provided the\n functions ``chr()`` and ``ord()`` implement a mapping between\n ASCII and EBCDIC, and string comparison preserves the ASCII\n order. Or perhaps someone can propose a better rule?)\n\n Unicode\n The items of a Unicode object are Unicode code units. A\n Unicode code unit is represented by a Unicode object of one\n item and can hold either a 16-bit or 32-bit value\n representing a Unicode ordinal (the maximum value for the\n ordinal is given in ``sys.maxunicode``, and depends on how\n Python is configured at compile time). Surrogate pairs may\n be present in the Unicode object, and will be reported as two\n separate items. The built-in functions ``unichr()`` and\n ``ord()`` convert between code units and nonnegative integers\n representing the Unicode ordinals as defined in the Unicode\n Standard 3.0. Conversion from and to other encodings are\n possible through the Unicode method ``encode()`` and the\n built-in function ``unicode()``.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and ``del`` (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in ``bytearray()`` constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module ``array`` provides an additional example of\n a mutable sequence type.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function ``len()``\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n ``set()`` constructor and can be modified afterwards by several\n methods, such as ``add()``.\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in ``frozenset()`` constructor. As a frozenset is\n immutable and *hashable*, it can be used again as an element of\n another set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation ``a[k]`` selects the item indexed by\n ``k`` from the mapping ``a``; this can be used in expressions and\n as the target of assignments or ``del`` statements. The built-in\n function ``len()`` returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``) then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the ``{...}``\n notation (see section *Dictionary displays*).\n\n The extension modules ``dbm``, ``gdbm``, and ``bsddb`` provide\n additional examples of mapping types.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +-------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +=========================+=================================+=============+\n | ``func_doc`` | The function\'s documentation | Writable |\n | | string, or ``None`` if | |\n | | unavailable | |\n +-------------------------+---------------------------------+-------------+\n | ``__doc__`` | Another way of spelling | Writable |\n | | ``func_doc`` | |\n +-------------------------+---------------------------------+-------------+\n | ``func_name`` | The function\'s name | Writable |\n +-------------------------+---------------------------------+-------------+\n | ``__name__`` | Another way of spelling | Writable |\n | | ``func_name`` | |\n +-------------------------+---------------------------------+-------------+\n | ``__module__`` | The name of the module the | Writable |\n | | function was defined in, or | |\n | | ``None`` if unavailable. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_defaults`` | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or ``None`` if no arguments | |\n | | have a default value | |\n +-------------------------+---------------------------------+-------------+\n | ``func_code`` | The code object representing | Writable |\n | | the compiled function body. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_globals`` | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_dict`` | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_closure`` | ``None`` or a tuple of cells | Read-only |\n | | that contain bindings for the | |\n | | function\'s free variables. | |\n +-------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Changed in version 2.4: ``func_name`` is now writable.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n User-defined methods\n A user-defined method object combines a class, a class instance\n (or ``None``) and any callable object (normally a user-defined\n function).\n\n Special read-only attributes: ``im_self`` is the class instance\n object, ``im_func`` is the function object; ``im_class`` is the\n class of ``im_self`` for bound methods or the class that asked\n for the method for unbound methods; ``__doc__`` is the method\'s\n documentation (same as ``im_func.__doc__``); ``__name__`` is the\n method name (same as ``im_func.__name__``); ``__module__`` is\n the name of the module the method was defined in, or ``None`` if\n unavailable.\n\n Changed in version 2.2: ``im_self`` used to refer to the class\n that defined the method.\n\n Changed in version 2.6: For Python 3 forward-compatibility,\n ``im_func`` is also available as ``__func__``, and ``im_self``\n as ``__self__``.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object, an unbound\n user-defined method object, or a class method object. When the\n attribute is a user-defined method object, a new method object\n is only created if the class from which it is being retrieved is\n the same as, or a derived class of, the class stored in the\n original method object; otherwise, the original method object is\n used as it is.\n\n When a user-defined method object is created by retrieving a\n user-defined function object from a class, its ``im_self``\n attribute is ``None`` and the method object is said to be\n unbound. When one is created by retrieving a user-defined\n function object from a class via one of its instances, its\n ``im_self`` attribute is the instance, and the method object is\n said to be bound. In either case, the new method\'s ``im_class``\n attribute is the class from which the retrieval takes place, and\n its ``im_func`` attribute is the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the ``im_func``\n attribute of the new instance is not the original method object\n but its ``im_func`` attribute.\n\n When a user-defined method object is created by retrieving a\n class method object from a class or instance, its ``im_self``\n attribute is the class itself (the same as the ``im_class``\n attribute), and its ``im_func`` attribute is the function object\n underlying the class method.\n\n When an unbound user-defined method object is called, the\n underlying function (``im_func``) is called, with the\n restriction that the first argument must be an instance of the\n proper class (``im_class``) or of a derived class thereof.\n\n When a bound user-defined method object is called, the\n underlying function (``im_func``) is called, inserting the class\n instance (``im_self``) in front of the argument list. For\n instance, when ``C`` is a class which contains a definition for\n a function ``f()``, and ``x`` is an instance of ``C``, calling\n ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``.\n\n When a user-defined method object is derived from a class method\n object, the "class instance" stored in ``im_self`` will actually\n be the class itself, so that calling either ``x.f(1)`` or\n ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n the underlying function.\n\n Note that the transformation from function object to (unbound or\n bound) method object happens each time the attribute is\n retrieved from the class or instance. In some cases, a fruitful\n optimization is to assign the attribute to a local variable and\n call that local variable. Also notice that this transformation\n only happens for user-defined functions; other callable objects\n (and all non-callable objects) are retrieved without\n transformation. It is also important to note that user-defined\n functions which are attributes of a class instance are not\n converted to bound methods; this *only* happens when the\n function is an attribute of the class.\n\n Generator functions\n A function or method which uses the ``yield`` statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s ``next()`` method will cause the function to\n execute until it provides a value using the ``yield`` statement.\n When the function executes a ``return`` statement or falls off\n the end, a ``StopIteration`` exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are ``len()`` and ``math.sin()``\n (``math`` is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: ``__doc__`` is the function\'s documentation\n string, or ``None`` if unavailable; ``__name__`` is the\n function\'s name; ``__self__`` is set to ``None`` (but see the\n next item); ``__module__`` is the name of the module the\n function was defined in or ``None`` if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n ``alist.append()``, assuming *alist* is a list object. In this\n case, the special read-only attribute ``__self__`` is set to the\n object denoted by *alist*.\n\n Class Types\n Class types, or "new-style classes," are callable. These\n objects normally act as factories for new instances of\n themselves, but variations are possible for class types that\n override ``__new__()``. The arguments of the call are passed to\n ``__new__()`` and, in the typical case, to ``__init__()`` to\n initialize the new instance.\n\n Classic Classes\n Class objects are described below. When a class object is\n called, a new class instance (also described below) is created\n and returned. This implies a call to the class\'s ``__init__()``\n method if it has one. Any arguments are passed on to the\n ``__init__()`` method. If there is no ``__init__()`` method,\n the class must be called without arguments.\n\n Class instances\n Class instances are described below. Class instances are\n callable only when the class has a ``__call__()`` method;\n ``x(arguments)`` is a shorthand for ``x.__call__(arguments)``.\n\nModules\n Modules are imported by the ``import`` statement (see section *The\n import statement*). A module object has a namespace implemented by\n a dictionary object (this is the dictionary referenced by the\n func_globals attribute of functions defined in the module).\n Attribute references are translated to lookups in this dictionary,\n e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n does not contain the code object used to initialize the module\n (since it isn\'t needed once the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n Special read-only attribute: ``__dict__`` is the module\'s namespace\n as a dictionary object.\n\n **CPython implementation detail:** Because of the way CPython\n clears module dictionaries, the module dictionary will be cleared\n when the module falls out of scope even if the dictionary still has\n live references. To avoid this, copy the dictionary or keep the\n module around while using its dictionary directly.\n\n Predefined (writable) attributes: ``__name__`` is the module\'s\n name; ``__doc__`` is the module\'s documentation string, or ``None``\n if unavailable; ``__file__`` is the pathname of the file from which\n the module was loaded, if it was loaded from a file. The\n ``__file__`` attribute is not present for C modules that are\n statically linked into the interpreter; for extension modules\n loaded dynamically from a shared library, it is the pathname of the\n shared library file.\n\nClasses\n Both class types (new-style classes) and class objects (old-\n style/classic classes) are typically created by class definitions\n (see section *Class definitions*). A class has a namespace\n implemented by a dictionary object. Class attribute references are\n translated to lookups in this dictionary, e.g., ``C.x`` is\n translated to ``C.__dict__["x"]`` (although for new-style classes\n in particular there are a number of hooks which allow for other\n means of locating attributes). When the attribute name is not found\n there, the attribute search continues in the base classes. For\n old-style classes, the search is depth-first, left-to-right in the\n order of occurrence in the base class list. New-style classes use\n the more complex C3 method resolution order which behaves correctly\n even in the presence of \'diamond\' inheritance structures where\n there are multiple inheritance paths leading back to a common\n ancestor. Additional details on the C3 MRO used by new-style\n classes can be found in the documentation accompanying the 2.3\n release at http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class ``C``, say) would yield\n a user-defined function object or an unbound user-defined method\n object whose associated class is either ``C`` or one of its base\n classes, it is transformed into an unbound user-defined method\n object whose ``im_class`` attribute is ``C``. When it would yield a\n class method object, it is transformed into a bound user-defined\n method object whose ``im_class`` and ``im_self`` attributes are\n both ``C``. When it would yield a static method object, it is\n transformed into the object wrapped by the static method object.\n See section *Implementing Descriptors* for another way in which\n attributes retrieved from a class may differ from those actually\n contained in its ``__dict__`` (note that only new-style classes\n support descriptors).\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: ``__name__`` is the class name; ``__module__``\n is the module name in which the class was defined; ``__dict__`` is\n the dictionary containing the class\'s namespace; ``__bases__`` is a\n tuple (possibly empty or a singleton) containing the base classes,\n in the order of their occurrence in the base class list;\n ``__doc__`` is the class\'s documentation string, or None if\n undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object or an unbound user-defined method object whose\n associated class is the class (call it ``C``) of the instance for\n which the attribute reference was initiated or one of its bases, it\n is transformed into a bound user-defined method object whose\n ``im_class`` attribute is ``C`` and whose ``im_self`` attribute is\n the instance. Static method and class method objects are also\n transformed, as if they had been retrieved from class ``C``; see\n above under "Classes". See section *Implementing Descriptors* for\n another way in which attributes of a class retrieved via its\n instances may differ from the objects actually stored in the\n class\'s ``__dict__``. If no class attribute is found, and the\n object\'s class has a ``__getattr__()`` method, that is called to\n satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n ``__setattr__()`` or ``__delattr__()`` method, this is called\n instead of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: ``__dict__`` is the attribute dictionary;\n ``__class__`` is the instance\'s class.\n\nFiles\n A file object represents an open file. File objects are created by\n the ``open()`` built-in function, and also by ``os.popen()``,\n ``os.fdopen()``, and the ``makefile()`` method of socket objects\n (and perhaps by other functions or methods provided by extension\n modules). The objects ``sys.stdin``, ``sys.stdout`` and\n ``sys.stderr`` are initialized to file objects corresponding to the\n interpreter\'s standard input, output and error streams. See *File\n Objects* for complete documentation of file objects.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: ``co_name`` gives the function\n name; ``co_argcount`` is the number of positional arguments\n (including arguments with default values); ``co_nlocals`` is the\n number of local variables used by the function (including\n arguments); ``co_varnames`` is a tuple containing the names of\n the local variables (starting with the argument names);\n ``co_cellvars`` is a tuple containing the names of local\n variables that are referenced by nested functions;\n ``co_freevars`` is a tuple containing the names of free\n variables; ``co_code`` is a string representing the sequence of\n bytecode instructions; ``co_consts`` is a tuple containing the\n literals used by the bytecode; ``co_names`` is a tuple\n containing the names used by the bytecode; ``co_filename`` is\n the filename from which the code was compiled;\n ``co_firstlineno`` is the first line number of the function;\n ``co_lnotab`` is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); ``co_stacksize`` is the required stack size\n (including local variables); ``co_flags`` is an integer encoding\n a number of flags for the interpreter.\n\n The following flag bits are defined for ``co_flags``: bit\n ``0x04`` is set if the function uses the ``*arguments`` syntax\n to accept an arbitrary number of positional arguments; bit\n ``0x08`` is set if the function uses the ``**keywords`` syntax\n to accept arbitrary keyword arguments; bit ``0x20`` is set if\n the function is a generator.\n\n Future feature declarations (``from __future__ import\n division``) also use bits in ``co_flags`` to indicate whether a\n code object was compiled with a particular feature enabled: bit\n ``0x2000`` is set if the function was compiled with future\n division enabled; bits ``0x10`` and ``0x1000`` were used in\n earlier versions of Python.\n\n Other bits in ``co_flags`` are reserved for internal use.\n\n If a code object represents a function, the first item in\n ``co_consts`` is the documentation string of the function, or\n ``None`` if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: ``f_back`` is to the previous\n stack frame (towards the caller), or ``None`` if this is the\n bottom stack frame; ``f_code`` is the code object being executed\n in this frame; ``f_locals`` is the dictionary used to look up\n local variables; ``f_globals`` is used for global variables;\n ``f_builtins`` is used for built-in (intrinsic) names;\n ``f_restricted`` is a flag indicating whether the function is\n executing in restricted execution mode; ``f_lasti`` gives the\n precise instruction (this is an index into the bytecode string\n of the code object).\n\n Special writable attributes: ``f_trace``, if not ``None``, is a\n function called at the start of each source code line (this is\n used by the debugger); ``f_exc_type``, ``f_exc_value``,\n ``f_exc_traceback`` represent the last exception raised in the\n parent frame provided another exception was ever raised in the\n current frame (in all other cases they are None); ``f_lineno``\n is the current line number of the frame --- writing to this from\n within a trace function jumps to the given line (only for the\n bottom-most frame). A debugger can implement a Jump command\n (aka Set Next Statement) by writing to f_lineno.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as ``sys.exc_traceback``,\n and also as the third item of the tuple returned by\n ``sys.exc_info()``. The latter is the preferred interface,\n since it works correctly when the program is using multiple\n threads. When the program contains no suitable handler, the\n stack trace is written (nicely formatted) to the standard error\n stream; if the interpreter is interactive, it is also made\n available to the user as ``sys.last_traceback``.\n\n Special read-only attributes: ``tb_next`` is the next level in\n the stack trace (towards the frame where the exception\n occurred), or ``None`` if there is no next level; ``tb_frame``\n points to the execution frame of the current level;\n ``tb_lineno`` gives the line number where the exception\n occurred; ``tb_lasti`` indicates the precise instruction. The\n line number and last instruction in the traceback may differ\n from the line number of its frame object if the exception\n occurred in a ``try`` statement with no matching except clause\n or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices when *extended slice\n syntax* is used. This is a slice using two colons, or multiple\n slices or ellipses separated by commas, e.g., ``a[i:j:step]``,\n ``a[i:j, k:l]``, or ``a[..., i:j]``. They are also created by\n the built-in ``slice()`` function.\n\n Special read-only attributes: ``start`` is the lower bound;\n ``stop`` is the upper bound; ``step`` is the step value; each is\n ``None`` if omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the extended slice that the slice\n object would describe if applied to a sequence of *length*\n items. It returns a tuple of three integers; respectively\n these are the *start* and *stop* indices and the *step* or\n stride length of the slice. Missing or out-of-bounds indices\n are handled in a manner consistent with regular slices.\n\n New in version 2.3.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n ``staticmethod()`` constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in ``classmethod()`` constructor.\n', + 'types': '\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.).\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name ``None``.\n It is used to signify the absence of a value in many situations,\n e.g., it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``NotImplemented``. Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``Ellipsis``. It is used to indicate the presence of the ``...``\n syntax in a slice. Its truth value is true.\n\n``numbers.Number``\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n ``numbers.Integral``\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are three types of integers:\n\n Plain integers\n These represent numbers in the range -2147483648 through\n 2147483647. (The range may be larger on machines with a\n larger natural word size, but not smaller.) When the result\n of an operation would fall outside this range, the result is\n normally returned as a long integer (in some cases, the\n exception ``OverflowError`` is raised instead). For the\n purpose of shift and mask operations, integers are assumed to\n have a binary, 2\'s complement notation using 32 or more bits,\n and hiding no bits from the user (i.e., all 4294967296\n different bit patterns correspond to different values).\n\n Long integers\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans\n These represent the truth values False and True. The two\n objects representing the values False and True are the only\n Boolean objects. The Boolean type is a subtype of plain\n integers, and Boolean values behave like the values 0 and 1,\n respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ``"False"`` or\n ``"True"`` are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers and the least surprises when\n switching between the plain and long integer domains. Any\n operation, if it yields a result in the plain integer domain,\n will yield the same result in the long integer domain or when\n using mixed operands. The switch between domains is transparent\n to the programmer.\n\n ``numbers.Real`` (``float``)\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n ``numbers.Complex``\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number ``z`` can be retrieved through the read-only\n attributes ``z.real`` and ``z.imag``.\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function ``len()`` returns the number of\n items of a sequence. When the length of a sequence is *n*, the\n index set contains the numbers 0, 1, ..., *n*-1. Item *i* of\n sequence *a* is selected by ``a[i]``.\n\n Sequences also support slicing: ``a[i:j]`` selects all items with\n index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n The items of a string are characters. There is no separate\n character type; a character is represented by a string of one\n item. Characters represent (at least) 8-bit bytes. The\n built-in functions ``chr()`` and ``ord()`` convert between\n characters and nonnegative integers representing the byte\n values. Bytes with the values 0-127 usually represent the\n corresponding ASCII values, but the interpretation of values\n is up to the program. The string data type is also used to\n represent arrays of bytes, e.g., to hold data read from a\n file.\n\n (On systems whose native character set is not ASCII, strings\n may use EBCDIC in their internal representation, provided the\n functions ``chr()`` and ``ord()`` implement a mapping between\n ASCII and EBCDIC, and string comparison preserves the ASCII\n order. Or perhaps someone can propose a better rule?)\n\n Unicode\n The items of a Unicode object are Unicode code units. A\n Unicode code unit is represented by a Unicode object of one\n item and can hold either a 16-bit or 32-bit value\n representing a Unicode ordinal (the maximum value for the\n ordinal is given in ``sys.maxunicode``, and depends on how\n Python is configured at compile time). Surrogate pairs may\n be present in the Unicode object, and will be reported as two\n separate items. The built-in functions ``unichr()`` and\n ``ord()`` convert between code units and nonnegative integers\n representing the Unicode ordinals as defined in the Unicode\n Standard 3.0. Conversion from and to other encodings are\n possible through the Unicode method ``encode()`` and the\n built-in function ``unicode()``.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and ``del`` (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in ``bytearray()`` constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module ``array`` provides an additional example of\n a mutable sequence type.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function ``len()``\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n ``set()`` constructor and can be modified afterwards by several\n methods, such as ``add()``.\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in ``frozenset()`` constructor. As a frozenset is\n immutable and *hashable*, it can be used again as an element of\n another set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation ``a[k]`` selects the item indexed by\n ``k`` from the mapping ``a``; this can be used in expressions and\n as the target of assignments or ``del`` statements. The built-in\n function ``len()`` returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``) then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the ``{...}``\n notation (see section *Dictionary displays*).\n\n The extension modules ``dbm``, ``gdbm``, and ``bsddb`` provide\n additional examples of mapping types.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +-------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +=========================+=================================+=============+\n | ``func_doc`` | The function\'s documentation | Writable |\n | | string, or ``None`` if | |\n | | unavailable | |\n +-------------------------+---------------------------------+-------------+\n | ``__doc__`` | Another way of spelling | Writable |\n | | ``func_doc`` | |\n +-------------------------+---------------------------------+-------------+\n | ``func_name`` | The function\'s name | Writable |\n +-------------------------+---------------------------------+-------------+\n | ``__name__`` | Another way of spelling | Writable |\n | | ``func_name`` | |\n +-------------------------+---------------------------------+-------------+\n | ``__module__`` | The name of the module the | Writable |\n | | function was defined in, or | |\n | | ``None`` if unavailable. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_defaults`` | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or ``None`` if no arguments | |\n | | have a default value | |\n +-------------------------+---------------------------------+-------------+\n | ``func_code`` | The code object representing | Writable |\n | | the compiled function body. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_globals`` | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_dict`` | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_closure`` | ``None`` or a tuple of cells | Read-only |\n | | that contain bindings for the | |\n | | function\'s free variables. | |\n +-------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Changed in version 2.4: ``func_name`` is now writable.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n User-defined methods\n A user-defined method object combines a class, a class instance\n (or ``None``) and any callable object (normally a user-defined\n function).\n\n Special read-only attributes: ``im_self`` is the class instance\n object, ``im_func`` is the function object; ``im_class`` is the\n class of ``im_self`` for bound methods or the class that asked\n for the method for unbound methods; ``__doc__`` is the method\'s\n documentation (same as ``im_func.__doc__``); ``__name__`` is the\n method name (same as ``im_func.__name__``); ``__module__`` is\n the name of the module the method was defined in, or ``None`` if\n unavailable.\n\n Changed in version 2.2: ``im_self`` used to refer to the class\n that defined the method.\n\n Changed in version 2.6: For Python 3 forward-compatibility,\n ``im_func`` is also available as ``__func__``, and ``im_self``\n as ``__self__``.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object, an unbound\n user-defined method object, or a class method object. When the\n attribute is a user-defined method object, a new method object\n is only created if the class from which it is being retrieved is\n the same as, or a derived class of, the class stored in the\n original method object; otherwise, the original method object is\n used as it is.\n\n When a user-defined method object is created by retrieving a\n user-defined function object from a class, its ``im_self``\n attribute is ``None`` and the method object is said to be\n unbound. When one is created by retrieving a user-defined\n function object from a class via one of its instances, its\n ``im_self`` attribute is the instance, and the method object is\n said to be bound. In either case, the new method\'s ``im_class``\n attribute is the class from which the retrieval takes place, and\n its ``im_func`` attribute is the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the ``im_func``\n attribute of the new instance is not the original method object\n but its ``im_func`` attribute.\n\n When a user-defined method object is created by retrieving a\n class method object from a class or instance, its ``im_self``\n attribute is the class itself, and its ``im_func`` attribute is\n the function object underlying the class method.\n\n When an unbound user-defined method object is called, the\n underlying function (``im_func``) is called, with the\n restriction that the first argument must be an instance of the\n proper class (``im_class``) or of a derived class thereof.\n\n When a bound user-defined method object is called, the\n underlying function (``im_func``) is called, inserting the class\n instance (``im_self``) in front of the argument list. For\n instance, when ``C`` is a class which contains a definition for\n a function ``f()``, and ``x`` is an instance of ``C``, calling\n ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``.\n\n When a user-defined method object is derived from a class method\n object, the "class instance" stored in ``im_self`` will actually\n be the class itself, so that calling either ``x.f(1)`` or\n ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n the underlying function.\n\n Note that the transformation from function object to (unbound or\n bound) method object happens each time the attribute is\n retrieved from the class or instance. In some cases, a fruitful\n optimization is to assign the attribute to a local variable and\n call that local variable. Also notice that this transformation\n only happens for user-defined functions; other callable objects\n (and all non-callable objects) are retrieved without\n transformation. It is also important to note that user-defined\n functions which are attributes of a class instance are not\n converted to bound methods; this *only* happens when the\n function is an attribute of the class.\n\n Generator functions\n A function or method which uses the ``yield`` statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s ``next()`` method will cause the function to\n execute until it provides a value using the ``yield`` statement.\n When the function executes a ``return`` statement or falls off\n the end, a ``StopIteration`` exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are ``len()`` and ``math.sin()``\n (``math`` is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: ``__doc__`` is the function\'s documentation\n string, or ``None`` if unavailable; ``__name__`` is the\n function\'s name; ``__self__`` is set to ``None`` (but see the\n next item); ``__module__`` is the name of the module the\n function was defined in or ``None`` if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n ``alist.append()``, assuming *alist* is a list object. In this\n case, the special read-only attribute ``__self__`` is set to the\n object denoted by *alist*.\n\n Class Types\n Class types, or "new-style classes," are callable. These\n objects normally act as factories for new instances of\n themselves, but variations are possible for class types that\n override ``__new__()``. The arguments of the call are passed to\n ``__new__()`` and, in the typical case, to ``__init__()`` to\n initialize the new instance.\n\n Classic Classes\n Class objects are described below. When a class object is\n called, a new class instance (also described below) is created\n and returned. This implies a call to the class\'s ``__init__()``\n method if it has one. Any arguments are passed on to the\n ``__init__()`` method. If there is no ``__init__()`` method,\n the class must be called without arguments.\n\n Class instances\n Class instances are described below. Class instances are\n callable only when the class has a ``__call__()`` method;\n ``x(arguments)`` is a shorthand for ``x.__call__(arguments)``.\n\nModules\n Modules are imported by the ``import`` statement (see section *The\n import statement*). A module object has a namespace implemented by\n a dictionary object (this is the dictionary referenced by the\n func_globals attribute of functions defined in the module).\n Attribute references are translated to lookups in this dictionary,\n e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n does not contain the code object used to initialize the module\n (since it isn\'t needed once the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n Special read-only attribute: ``__dict__`` is the module\'s namespace\n as a dictionary object.\n\n **CPython implementation detail:** Because of the way CPython\n clears module dictionaries, the module dictionary will be cleared\n when the module falls out of scope even if the dictionary still has\n live references. To avoid this, copy the dictionary or keep the\n module around while using its dictionary directly.\n\n Predefined (writable) attributes: ``__name__`` is the module\'s\n name; ``__doc__`` is the module\'s documentation string, or ``None``\n if unavailable; ``__file__`` is the pathname of the file from which\n the module was loaded, if it was loaded from a file. The\n ``__file__`` attribute is not present for C modules that are\n statically linked into the interpreter; for extension modules\n loaded dynamically from a shared library, it is the pathname of the\n shared library file.\n\nClasses\n Both class types (new-style classes) and class objects (old-\n style/classic classes) are typically created by class definitions\n (see section *Class definitions*). A class has a namespace\n implemented by a dictionary object. Class attribute references are\n translated to lookups in this dictionary, e.g., ``C.x`` is\n translated to ``C.__dict__["x"]`` (although for new-style classes\n in particular there are a number of hooks which allow for other\n means of locating attributes). When the attribute name is not found\n there, the attribute search continues in the base classes. For\n old-style classes, the search is depth-first, left-to-right in the\n order of occurrence in the base class list. New-style classes use\n the more complex C3 method resolution order which behaves correctly\n even in the presence of \'diamond\' inheritance structures where\n there are multiple inheritance paths leading back to a common\n ancestor. Additional details on the C3 MRO used by new-style\n classes can be found in the documentation accompanying the 2.3\n release at http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class ``C``, say) would yield\n a user-defined function object or an unbound user-defined method\n object whose associated class is either ``C`` or one of its base\n classes, it is transformed into an unbound user-defined method\n object whose ``im_class`` attribute is ``C``. When it would yield a\n class method object, it is transformed into a bound user-defined\n method object whose ``im_self`` attribute is ``C``. When it would\n yield a static method object, it is transformed into the object\n wrapped by the static method object. See section *Implementing\n Descriptors* for another way in which attributes retrieved from a\n class may differ from those actually contained in its ``__dict__``\n (note that only new-style classes support descriptors).\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: ``__name__`` is the class name; ``__module__``\n is the module name in which the class was defined; ``__dict__`` is\n the dictionary containing the class\'s namespace; ``__bases__`` is a\n tuple (possibly empty or a singleton) containing the base classes,\n in the order of their occurrence in the base class list;\n ``__doc__`` is the class\'s documentation string, or None if\n undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object or an unbound user-defined method object whose\n associated class is the class (call it ``C``) of the instance for\n which the attribute reference was initiated or one of its bases, it\n is transformed into a bound user-defined method object whose\n ``im_class`` attribute is ``C`` and whose ``im_self`` attribute is\n the instance. Static method and class method objects are also\n transformed, as if they had been retrieved from class ``C``; see\n above under "Classes". See section *Implementing Descriptors* for\n another way in which attributes of a class retrieved via its\n instances may differ from the objects actually stored in the\n class\'s ``__dict__``. If no class attribute is found, and the\n object\'s class has a ``__getattr__()`` method, that is called to\n satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n ``__setattr__()`` or ``__delattr__()`` method, this is called\n instead of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: ``__dict__`` is the attribute dictionary;\n ``__class__`` is the instance\'s class.\n\nFiles\n A file object represents an open file. File objects are created by\n the ``open()`` built-in function, and also by ``os.popen()``,\n ``os.fdopen()``, and the ``makefile()`` method of socket objects\n (and perhaps by other functions or methods provided by extension\n modules). The objects ``sys.stdin``, ``sys.stdout`` and\n ``sys.stderr`` are initialized to file objects corresponding to the\n interpreter\'s standard input, output and error streams. See *File\n Objects* for complete documentation of file objects.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: ``co_name`` gives the function\n name; ``co_argcount`` is the number of positional arguments\n (including arguments with default values); ``co_nlocals`` is the\n number of local variables used by the function (including\n arguments); ``co_varnames`` is a tuple containing the names of\n the local variables (starting with the argument names);\n ``co_cellvars`` is a tuple containing the names of local\n variables that are referenced by nested functions;\n ``co_freevars`` is a tuple containing the names of free\n variables; ``co_code`` is a string representing the sequence of\n bytecode instructions; ``co_consts`` is a tuple containing the\n literals used by the bytecode; ``co_names`` is a tuple\n containing the names used by the bytecode; ``co_filename`` is\n the filename from which the code was compiled;\n ``co_firstlineno`` is the first line number of the function;\n ``co_lnotab`` is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); ``co_stacksize`` is the required stack size\n (including local variables); ``co_flags`` is an integer encoding\n a number of flags for the interpreter.\n\n The following flag bits are defined for ``co_flags``: bit\n ``0x04`` is set if the function uses the ``*arguments`` syntax\n to accept an arbitrary number of positional arguments; bit\n ``0x08`` is set if the function uses the ``**keywords`` syntax\n to accept arbitrary keyword arguments; bit ``0x20`` is set if\n the function is a generator.\n\n Future feature declarations (``from __future__ import\n division``) also use bits in ``co_flags`` to indicate whether a\n code object was compiled with a particular feature enabled: bit\n ``0x2000`` is set if the function was compiled with future\n division enabled; bits ``0x10`` and ``0x1000`` were used in\n earlier versions of Python.\n\n Other bits in ``co_flags`` are reserved for internal use.\n\n If a code object represents a function, the first item in\n ``co_consts`` is the documentation string of the function, or\n ``None`` if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: ``f_back`` is to the previous\n stack frame (towards the caller), or ``None`` if this is the\n bottom stack frame; ``f_code`` is the code object being executed\n in this frame; ``f_locals`` is the dictionary used to look up\n local variables; ``f_globals`` is used for global variables;\n ``f_builtins`` is used for built-in (intrinsic) names;\n ``f_restricted`` is a flag indicating whether the function is\n executing in restricted execution mode; ``f_lasti`` gives the\n precise instruction (this is an index into the bytecode string\n of the code object).\n\n Special writable attributes: ``f_trace``, if not ``None``, is a\n function called at the start of each source code line (this is\n used by the debugger); ``f_exc_type``, ``f_exc_value``,\n ``f_exc_traceback`` represent the last exception raised in the\n parent frame provided another exception was ever raised in the\n current frame (in all other cases they are None); ``f_lineno``\n is the current line number of the frame --- writing to this from\n within a trace function jumps to the given line (only for the\n bottom-most frame). A debugger can implement a Jump command\n (aka Set Next Statement) by writing to f_lineno.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as ``sys.exc_traceback``,\n and also as the third item of the tuple returned by\n ``sys.exc_info()``. The latter is the preferred interface,\n since it works correctly when the program is using multiple\n threads. When the program contains no suitable handler, the\n stack trace is written (nicely formatted) to the standard error\n stream; if the interpreter is interactive, it is also made\n available to the user as ``sys.last_traceback``.\n\n Special read-only attributes: ``tb_next`` is the next level in\n the stack trace (towards the frame where the exception\n occurred), or ``None`` if there is no next level; ``tb_frame``\n points to the execution frame of the current level;\n ``tb_lineno`` gives the line number where the exception\n occurred; ``tb_lasti`` indicates the precise instruction. The\n line number and last instruction in the traceback may differ\n from the line number of its frame object if the exception\n occurred in a ``try`` statement with no matching except clause\n or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices when *extended slice\n syntax* is used. This is a slice using two colons, or multiple\n slices or ellipses separated by commas, e.g., ``a[i:j:step]``,\n ``a[i:j, k:l]``, or ``a[..., i:j]``. They are also created by\n the built-in ``slice()`` function.\n\n Special read-only attributes: ``start`` is the lower bound;\n ``stop`` is the upper bound; ``step`` is the step value; each is\n ``None`` if omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the extended slice that the slice\n object would describe if applied to a sequence of *length*\n items. It returns a tuple of three integers; respectively\n these are the *start* and *stop* indices and the *step* or\n stride length of the slice. Missing or out-of-bounds indices\n are handled in a manner consistent with regular slices.\n\n New in version 2.3.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n ``staticmethod()`` constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in ``classmethod()`` constructor.\n', 'typesfunctions': '\nFunctions\n*********\n\nFunction objects are created by function definitions. The only\noperation on a function object is to call it: ``func(argument-list)``.\n\nThere are really two flavors of function objects: built-in functions\nand user-defined functions. Both support the same operation (to call\nthe function), but the implementation is different, hence the\ndifferent object types.\n\nSee *Function definitions* for more information.\n', 'typesmapping': '\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry. (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict(**kwarg)\nclass class dict(mapping, **kwarg)\nclass class dict(iterable, **kwarg)\n\n Return a new dictionary initialized from an optional positional\n argument and a possibly empty set of keyword arguments.\n\n If no positional argument is given, an empty dictionary is created.\n If a positional argument is given and it is a mapping object, a\n dictionary is created with the same key-value pairs as the mapping\n object. Otherwise, the positional argument must be an *iterator*\n object. Each item in the iterable must itself be an iterator with\n exactly two objects. The first object of each item becomes a key\n in the new dictionary, and the second object the corresponding\n value. If a key occurs more than once, the last value for that key\n becomes the corresponding value in the new dictionary.\n\n If keyword arguments are given, the keyword arguments and their\n values are added to the dictionary created from the positional\n argument. If a key being added is already present, the value from\n the keyword argument replaces the value from the positional\n argument.\n\n To illustrate, the following examples all return a dictionary equal\n to ``{"one": 1, "two": 2, "three": 3}``:\n\n >>> a = dict(one=1, two=2, three=3)\n >>> b = {\'one\': 1, \'two\': 2, \'three\': 3}\n >>> c = dict(zip([\'one\', \'two\', \'three\'], [1, 2, 3]))\n >>> d = dict([(\'two\', 2), (\'one\', 1), (\'three\', 3)])\n >>> e = dict({\'three\': 3, \'one\': 1, \'two\': 2})\n >>> a == b == c == d == e\n True\n\n Providing keyword arguments as in the first example only works for\n keys that are valid Python identifiers. Otherwise, any valid keys\n can be used.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for building a dictionary from\n keyword arguments added.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a ``KeyError`` if\n *key* is not in the map.\n\n New in version 2.5: If a subclass of dict defines a method\n ``__missing__()``, if the key *key* is not present, the\n ``d[key]`` operation calls that method with the key *key* as\n argument. The ``d[key]`` operation then returns or raises\n whatever is returned or raised by the ``__missing__(key)`` call\n if the key is not present. No other operations or methods invoke\n ``__missing__()``. If ``__missing__()`` is not defined,\n ``KeyError`` is raised. ``__missing__()`` must be a method; it\n cannot be an instance variable. For an example, see\n ``collections.defaultdict``.\n\n d[key] = value\n\n Set ``d[key]`` to *value*.\n\n del d[key]\n\n Remove ``d[key]`` from *d*. Raises a ``KeyError`` if *key* is\n not in the map.\n\n key in d\n\n Return ``True`` if *d* has a key *key*, else ``False``.\n\n New in version 2.2.\n\n key not in d\n\n Equivalent to ``not key in d``.\n\n New in version 2.2.\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for ``iterkeys()``.\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n ``fromkeys()`` is a class method that returns a new dictionary.\n *value* defaults to ``None``.\n\n New in version 2.3.\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to ``None``,\n so that this method never raises a ``KeyError``.\n\n has_key(key)\n\n Test for the presence of *key* in the dictionary. ``has_key()``\n is deprecated in favor of ``key in d``.\n\n items()\n\n Return a copy of the dictionary\'s list of ``(key, value)``\n pairs.\n\n **CPython implementation detail:** Keys and values are listed in\n an arbitrary order which is non-random, varies across Python\n implementations, and depends on the dictionary\'s history of\n insertions and deletions.\n\n If ``items()``, ``keys()``, ``values()``, ``iteritems()``,\n ``iterkeys()``, and ``itervalues()`` are called with no\n intervening modifications to the dictionary, the lists will\n directly correspond. This allows the creation of ``(value,\n key)`` pairs using ``zip()``: ``pairs = zip(d.values(),\n d.keys())``. The same relationship holds for the ``iterkeys()``\n and ``itervalues()`` methods: ``pairs = zip(d.itervalues(),\n d.iterkeys())`` provides the same value for ``pairs``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.iteritems()]``.\n\n iteritems()\n\n Return an iterator over the dictionary\'s ``(key, value)`` pairs.\n See the note for ``dict.items()``.\n\n Using ``iteritems()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n iterkeys()\n\n Return an iterator over the dictionary\'s keys. See the note for\n ``dict.items()``.\n\n Using ``iterkeys()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n itervalues()\n\n Return an iterator over the dictionary\'s values. See the note\n for ``dict.items()``.\n\n Using ``itervalues()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n keys()\n\n Return a copy of the dictionary\'s list of keys. See the note\n for ``dict.items()``.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a ``KeyError`` is raised.\n\n New in version 2.3.\n\n popitem()\n\n Remove and return an arbitrary ``(key, value)`` pair from the\n dictionary.\n\n ``popitem()`` is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling ``popitem()`` raises a ``KeyError``.\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to ``None``.\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return ``None``.\n\n ``update()`` accepts either another dictionary object or an\n iterable of key/value pairs (as tuples or other iterables of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: ``d.update(red=1,\n blue=2)``.\n\n Changed in version 2.4: Allowed the argument to be an iterable\n of key/value pairs and allowed keyword arguments.\n\n values()\n\n Return a copy of the dictionary\'s list of values. See the note\n for ``dict.items()``.\n\n viewitems()\n\n Return a new view of the dictionary\'s items (``(key, value)``\n pairs). See below for documentation of view objects.\n\n New in version 2.7.\n\n viewkeys()\n\n Return a new view of the dictionary\'s keys. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n viewvalues()\n\n Return a new view of the dictionary\'s values. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.viewkeys()``, ``dict.viewvalues()`` and\n``dict.viewitems()`` are *view objects*. They provide a dynamic view\non the dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of ``(key, value)``) in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of ``(value, key)`` pairs\n using ``zip()``: ``pairs = zip(d.values(), d.keys())``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.items()]``.\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n Return ``True`` if *x* is in the underlying dictionary\'s keys,\n values or items (in the latter case, *x* should be a ``(key,\n value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that (key, value) pairs are unique and\nhashable, then the items view is also set-like. (Values views are not\ntreated as set-like since the entries are generally not unique.) Then\nthese set operations are available ("other" refers either to another\nview or a set):\n\ndictview & other\n\n Return the intersection of the dictview and the other object as a\n new set.\n\ndictview | other\n\n Return the union of the dictview and the other object as a new set.\n\ndictview - other\n\n Return the difference between the dictview and the other object\n (all elements in *dictview* that aren\'t in *other*) as a new set.\n\ndictview ^ other\n\n Return the symmetric difference (all elements either in *dictview*\n or *other*, but not in both) of the dictview and the other object\n as a new set.\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.viewkeys()\n >>> values = dishes.viewvalues()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n', 'typesmethods': '\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as ``append()`` on\nlists) and class instance methods. Built-in methods are described\nwith the types that support them.\n\nThe implementation adds two special read-only attributes to class\ninstance methods: ``m.im_self`` is the object on which the method\noperates, and ``m.im_func`` is the function implementing the method.\nCalling ``m(arg-1, arg-2, ..., arg-n)`` is completely equivalent to\ncalling ``m.im_func(m.im_self, arg-1, arg-2, ..., arg-n)``.\n\nClass instance methods are either *bound* or *unbound*, referring to\nwhether the method was accessed through an instance or a class,\nrespectively. When a method is unbound, its ``im_self`` attribute\nwill be ``None`` and if called, an explicit ``self`` object must be\npassed as the first argument. In this case, ``self`` must be an\ninstance of the unbound method\'s class (or a subclass of that class),\notherwise a ``TypeError`` is raised.\n\nLike function objects, methods objects support getting arbitrary\nattributes. However, since method attributes are actually stored on\nthe underlying function object (``meth.im_func``), setting method\nattributes on either bound or unbound methods is disallowed.\nAttempting to set an attribute on a method results in an\n``AttributeError`` being raised. In order to set a method attribute,\nyou need to explicitly set it on the underlying function object:\n\n >>> class C:\n ... def method(self):\n ... pass\n ...\n >>> c = C()\n >>> c.method.whoami = \'my name is method\' # can\'t set on the method\n Traceback (most recent call last):\n File "", line 1, in \n AttributeError: \'instancemethod\' object has no attribute \'whoami\'\n >>> c.method.im_func.whoami = \'my name is method\'\n >>> c.method.whoami\n \'my name is method\'\n\nSee *The standard type hierarchy* for more information.\n', 'typesmodules': "\nModules\n*******\n\nThe only special operation on a module is attribute access:\n``m.name``, where *m* is a module and *name* accesses a name defined\nin *m*'s symbol table. Module attributes can be assigned to. (Note\nthat the ``import`` statement is not, strictly speaking, an operation\non a module object; ``import foo`` does not require a module object\nnamed *foo* to exist, rather it requires an (external) *definition*\nfor a module named *foo* somewhere.)\n\nA special attribute of every module is ``__dict__``. This is the\ndictionary containing the module's symbol table. Modifying this\ndictionary will actually change the module's symbol table, but direct\nassignment to the ``__dict__`` attribute is not possible (you can\nwrite ``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but\nyou can't write ``m.__dict__ = {}``). Modifying ``__dict__`` directly\nis not recommended.\n\nModules built into the interpreter are written like this: ````. If loaded from a file, they are written as\n````.\n", - 'typesseq': '\nSequence Types --- ``str``, ``unicode``, ``list``, ``tuple``, ``bytearray``, ``buffer``, ``xrange``\n***************************************************************************************************\n\nThere are seven sequence types: strings, Unicode strings, lists,\ntuples, bytearrays, buffers, and xrange objects.\n\nFor other containers see the built in ``dict`` and ``set`` classes,\nand the ``collections`` module.\n\nString literals are written in single or double quotes: ``\'xyzzy\'``,\n``"frobozz"``. See *String literals* for more about string literals.\nUnicode strings are much like strings, but are specified in the syntax\nusing a preceding ``\'u\'`` character: ``u\'abc\'``, ``u"def"``. In\naddition to the functionality described here, there are also string-\nspecific methods described in the *String Methods* section. Lists are\nconstructed with square brackets, separating items with commas: ``[a,\nb, c]``. Tuples are constructed by the comma operator (not within\nsquare brackets), with or without enclosing parentheses, but an empty\ntuple must have the enclosing parentheses, such as ``a, b, c`` or\n``()``. A single item tuple must have a trailing comma, such as\n``(d,)``.\n\nBytearray objects are created with the built-in function\n``bytearray()``.\n\nBuffer objects are not directly supported by Python syntax, but can be\ncreated by calling the built-in function ``buffer()``. They don\'t\nsupport concatenation or repetition.\n\nObjects of type xrange are similar to buffers in that there is no\nspecific syntax to create them, but they are created using the\n``xrange()`` function. They don\'t support slicing, concatenation or\nrepetition, and using ``in``, ``not in``, ``min()`` or ``max()`` on\nthem is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)`` | index of the first occurence of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)`` | total number of occurences of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must compare\nequal and the two sequences must be of the same type and have the same\nlength. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string or Unicode string object the ``in`` and ``not\n in`` operations act like a substring test. In Python versions\n before 2.3, *x* had to be a string of length 1. In Python 2.3 and\n beyond, *x* may be a string of any length.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n Changed in version 2.4: Formerly, string concatenation never\n occurred in-place.\n\n\nString Methods\n==============\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters [4] in the string are lowercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters [4] in the string are uppercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string with all the cased characters [4]\n converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified or ``-1``, then there is\n no limit on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. This method uses the *universal newlines* approach to\n splitting lines. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\n For example, ``\'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()`` returns\n ``[\'ab c\', \'\', \'de fg\', \'kl\']``, while the same call with\n ``splitlines(True)`` returns ``[\'ab c\\n\', \'\\n\', \'de fg\\r\',\n \'kl\\r\\n\']``.\n\n Unlike ``split()`` when a delimiter string *sep* is given, this\n method returns an empty list for the empty string, and a terminal\n line break does not result in an extra line.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n ... lambda mo: mo.group(0)[0].upper() +\n ... mo.group(0)[1:].lower(),\n ... s)\n ...\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string with all the cased characters [4]\n converted to uppercase. Note that ``str.upper().isupper()`` might\n be ``False`` if ``s`` contains uncased characters or if the Unicode\n category of the resulting character(s) is not "Lu" (Letter,\n uppercase), but e.g. "Lt" (Letter, titlecase).\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than or equal to ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that can be used to form decimal-radix numbers,\n e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\n\nString Formatting Operations\n============================\n\nString and Unicode objects have one unique built-in operation: the\n``%`` operator (modulo). This is also known as the string\n*formatting* or *interpolation* operator. Given ``format % values``\n(where *format* is a string or Unicode object), ``%`` conversion\nspecifications in *format* are replaced with zero or more elements of\n*values*. The effect is similar to the using ``sprintf()`` in the C\nlanguage. If *format* is a Unicode object, or if any of the objects\nbeing converted using the ``%s`` conversion are Unicode objects, the\nresult will also be a Unicode object.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [5] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print \'%(language)s has %(number)03d quote types.\' % \\\n... {"language": "Python", "number": 2}\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using ``repr()``). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | (6) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The ``%r`` conversion was added in Python 2.0.\n\n The precision determines the maximal number of characters used.\n\n6. If the object or format provided is a ``unicode`` string, the\n resulting string will also be ``unicode``.\n\n The precision determines the maximal number of characters used.\n\n7. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 2.7: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nXRange Type\n===========\n\nThe ``xrange`` type is an immutable sequence which is commonly used\nfor looping. The advantage of the ``xrange`` type is that an\n``xrange`` object will always take the same amount of memory, no\nmatter the size of the range it represents. There are no consistent\nperformance advantages.\n\nXRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn\'t have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don\'t return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n', + 'typesseq': '\nSequence Types --- ``str``, ``unicode``, ``list``, ``tuple``, ``bytearray``, ``buffer``, ``xrange``\n***************************************************************************************************\n\nThere are seven sequence types: strings, Unicode strings, lists,\ntuples, bytearrays, buffers, and xrange objects.\n\nFor other containers see the built in ``dict`` and ``set`` classes,\nand the ``collections`` module.\n\nString literals are written in single or double quotes: ``\'xyzzy\'``,\n``"frobozz"``. See *String literals* for more about string literals.\nUnicode strings are much like strings, but are specified in the syntax\nusing a preceding ``\'u\'`` character: ``u\'abc\'``, ``u"def"``. In\naddition to the functionality described here, there are also string-\nspecific methods described in the *String Methods* section. Lists are\nconstructed with square brackets, separating items with commas: ``[a,\nb, c]``. Tuples are constructed by the comma operator (not within\nsquare brackets), with or without enclosing parentheses, but an empty\ntuple must have the enclosing parentheses, such as ``a, b, c`` or\n``()``. A single item tuple must have a trailing comma, such as\n``(d,)``.\n\nBytearray objects are created with the built-in function\n``bytearray()``.\n\nBuffer objects are not directly supported by Python syntax, but can be\ncreated by calling the built-in function ``buffer()``. They don\'t\nsupport concatenation or repetition.\n\nObjects of type xrange are similar to buffers in that there is no\nspecific syntax to create them, but they are created using the\n``xrange()`` function. They don\'t support slicing, concatenation or\nrepetition, and using ``in``, ``not in``, ``min()`` or ``max()`` on\nthem is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)`` | index of the first occurence of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)`` | total number of occurences of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must compare\nequal and the two sequences must be of the same type and have the same\nlength. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string or Unicode string object the ``in`` and ``not\n in`` operations act like a substring test. In Python versions\n before 2.3, *x* had to be a string of length 1. In Python 2.3 and\n beyond, *x* may be a string of any length.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n Changed in version 2.4: Formerly, string concatenation never\n occurred in-place.\n\n\nString Methods\n==============\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. Tab positions occur every *tabsize* characters\n (default is 8, giving tab positions at columns 0, 8, 16 and so on).\n To expand the string, the current column is set to zero and the\n string is examined character by character. If the character is a\n tab (``\\t``), one or more space characters are inserted in the\n result until the current column is equal to the next tab position.\n (The tab character itself is not copied.) If the character is a\n newline (``\\n``) or return (``\\r``), it is copied and the current\n column is reset to zero. Any other character is copied unchanged\n and the current column is incremented by one regardless of how the\n character is represented when printed.\n\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs()\n \'01 012 0123 01234\'\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs(4)\n \'01 012 0123 01234\'\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters [4] in the string are lowercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters [4] in the string are uppercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string with all the cased characters [4]\n converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified or ``-1``, then there is\n no limit on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. This method uses the *universal newlines* approach to\n splitting lines. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\n For example, ``\'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()`` returns\n ``[\'ab c\', \'\', \'de fg\', \'kl\']``, while the same call with\n ``splitlines(True)`` returns ``[\'ab c\\n\', \'\\n\', \'de fg\\r\',\n \'kl\\r\\n\']``.\n\n Unlike ``split()`` when a delimiter string *sep* is given, this\n method returns an empty list for the empty string, and a terminal\n line break does not result in an extra line.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n ... lambda mo: mo.group(0)[0].upper() +\n ... mo.group(0)[1:].lower(),\n ... s)\n ...\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string with all the cased characters [4]\n converted to uppercase. Note that ``str.upper().isupper()`` might\n be ``False`` if ``s`` contains uncased characters or if the Unicode\n category of the resulting character(s) is not "Lu" (Letter,\n uppercase), but e.g. "Lt" (Letter, titlecase).\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than or equal to ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that can be used to form decimal-radix numbers,\n e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\n\nString Formatting Operations\n============================\n\nString and Unicode objects have one unique built-in operation: the\n``%`` operator (modulo). This is also known as the string\n*formatting* or *interpolation* operator. Given ``format % values``\n(where *format* is a string or Unicode object), ``%`` conversion\nspecifications in *format* are replaced with zero or more elements of\n*values*. The effect is similar to the using ``sprintf()`` in the C\nlanguage. If *format* is a Unicode object, or if any of the objects\nbeing converted using the ``%s`` conversion are Unicode objects, the\nresult will also be a Unicode object.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [5] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print \'%(language)s has %(number)03d quote types.\' % \\\n... {"language": "Python", "number": 2}\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using *repr()*). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | (6) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The ``%r`` conversion was added in Python 2.0.\n\n The precision determines the maximal number of characters used.\n\n6. If the object or format provided is a ``unicode`` string, the\n resulting string will also be ``unicode``.\n\n The precision determines the maximal number of characters used.\n\n7. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 2.7: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nXRange Type\n===========\n\nThe ``xrange`` type is an immutable sequence which is commonly used\nfor looping. The advantage of the ``xrange`` type is that an\n``xrange`` object will always take the same amount of memory, no\nmatter the size of the range it represents. There are no consistent\nperformance advantages.\n\nXRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn\'t have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don\'t return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n', 'typesseq-mutable': "\nMutable Sequence Types\n**********************\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn't have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don't return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n", 'unary': '\nUnary arithmetic and bitwise operations\n***************************************\n\nAll unary arithmetic and bitwise operations have the same priority:\n\n u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n\nThe unary ``-`` (minus) operator yields the negation of its numeric\nargument.\n\nThe unary ``+`` (plus) operator yields its numeric argument unchanged.\n\nThe unary ``~`` (invert) operator yields the bitwise inversion of its\nplain or long integer argument. The bitwise inversion of ``x`` is\ndefined as ``-(x+1)``. It only applies to integral numbers.\n\nIn all three cases, if the argument does not have the proper type, a\n``TypeError`` exception is raised.\n', 'while': '\nThe ``while`` statement\n***********************\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n', 'with': '\nThe ``with`` statement\n**********************\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the ``with_item``)\n is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', - 'yield': '\nThe ``yield`` statement\n***********************\n\n yield_stmt ::= yield_expression\n\nThe ``yield`` statement is only used when defining a generator\nfunction, and is only used in the body of the generator function.\nUsing a ``yield`` statement in a function definition is sufficient to\ncause that definition to create a generator function instead of a\nnormal function.\n\nWhen a generator function is called, it returns an iterator known as a\ngenerator iterator, or more commonly, a generator. The body of the\ngenerator function is executed by calling the generator\'s ``next()``\nmethod repeatedly until it raises an exception.\n\nWhen a ``yield`` statement is executed, the state of the generator is\nfrozen and the value of ``expression_list`` is returned to\n``next()``\'s caller. By "frozen" we mean that all local state is\nretained, including the current bindings of local variables, the\ninstruction pointer, and the internal evaluation stack: enough\ninformation is saved so that the next time ``next()`` is invoked, the\nfunction can proceed exactly as if the ``yield`` statement were just\nanother external call.\n\nAs of Python version 2.5, the ``yield`` statement is now allowed in\nthe ``try`` clause of a ``try`` ... ``finally`` construct. If the\ngenerator is not resumed before it is finalized (by reaching a zero\nreference count or by being garbage collected), the generator-\niterator\'s ``close()`` method will be called, allowing any pending\n``finally`` clauses to execute.\n\nNote: In Python 2.2, the ``yield`` statement was only allowed when the\n ``generators`` feature has been enabled. This ``__future__`` import\n statement was used to enable the feature:\n\n from __future__ import generators\n\nSee also:\n\n **PEP 0255** - Simple Generators\n The proposal for adding generators and the ``yield`` statement\n to Python.\n\n **PEP 0342** - Coroutines via Enhanced Generators\n The proposal that, among other generator enhancements, proposed\n allowing ``yield`` to appear inside a ``try`` ... ``finally``\n block.\n'} + 'yield': '\nThe ``yield`` statement\n***********************\n\n yield_stmt ::= yield_expression\n\nThe ``yield`` statement is only used when defining a generator\nfunction, and is only used in the body of the generator function.\nUsing a ``yield`` statement in a function definition is sufficient to\ncause that definition to create a generator function instead of a\nnormal function.\n\nWhen a generator function is called, it returns an iterator known as a\ngenerator iterator, or more commonly, a generator. The body of the\ngenerator function is executed by calling the generator\'s ``next()``\nmethod repeatedly until it raises an exception.\n\nWhen a ``yield`` statement is executed, the state of the generator is\nfrozen and the value of ``expression_list`` is returned to\n``next()``\'s caller. By "frozen" we mean that all local state is\nretained, including the current bindings of local variables, the\ninstruction pointer, and the internal evaluation stack: enough\ninformation is saved so that the next time ``next()`` is invoked, the\nfunction can proceed exactly as if the ``yield`` statement were just\nanother external call.\n\nAs of Python version 2.5, the ``yield`` statement is now allowed in\nthe ``try`` clause of a ``try`` ... ``finally`` construct. If the\ngenerator is not resumed before it is finalized (by reaching a zero\nreference count or by being garbage collected), the generator-\niterator\'s ``close()`` method will be called, allowing any pending\n``finally`` clauses to execute.\n\nFor full details of ``yield`` semantics, refer to the *Yield\nexpressions* section.\n\nNote: In Python 2.2, the ``yield`` statement was only allowed when the\n ``generators`` feature has been enabled. This ``__future__`` import\n statement was used to enable the feature:\n\n from __future__ import generators\n\nSee also:\n\n **PEP 0255** - Simple Generators\n The proposal for adding generators and the ``yield`` statement\n to Python.\n\n **PEP 0342** - Coroutines via Enhanced Generators\n The proposal that, among other generator enhancements, proposed\n allowing ``yield`` to appear inside a ``try`` ... ``finally``\n block.\n'} From cd96af998c365afa1dbb20add824b563c163790d Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 11 May 2013 22:31:46 -0500 Subject: [PATCH 123/480] Added tag v2.7.5 for changeset ab05e7dd2788 From c07724067e91623c1fbf3bab5d6e118e0cf43c8a Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 11 May 2013 22:36:05 -0500 Subject: [PATCH 124/480] onto 2.7.6 --- Include/patchlevel.h | 2 +- Misc/NEWS | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index bdf179e6241..07b79e41fb5 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -27,7 +27,7 @@ #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "2.7.5" +#define PY_VERSION "2.7.5+" /*--end constants--*/ /* Subversion Revision number of this file (not of the repository). Empty diff --git a/Misc/NEWS b/Misc/NEWS index 8f9739022e9..16023468809 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1,6 +1,18 @@ Python News +++++++++++ +What's New in Python 2.7.6? +=========================== + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +Library +------- + + What's New in Python 2.7.5? =========================== From 3ce682c8069b6d2ed4c3a5c8ae369accd623a170 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 12 May 2013 17:29:34 +0300 Subject: [PATCH 125/480] Issue #17606: Fixed support of encoded byte strings in the XMLGenerator characters() and ignorableWhitespace() methods. Original patch by Sebastian Ortiz Vasquez. --- Lib/test/test_sax.py | 20 ++++++++++++++++++++ Lib/xml/sax/saxutils.py | 8 ++++++-- Misc/ACKS | 1 + Misc/NEWS | 4 ++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index 3a20eac6b12..86638a2665d 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -284,6 +284,26 @@ def test_xmlgen_ignorable(self): self.assertEqual(result.getvalue(), start + " ") + def test_xmlgen_encoding_bytes(self): + encodings = ('iso-8859-15', 'utf-8', + 'utf-16be', 'utf-16le', + 'utf-32be', 'utf-32le') + for encoding in encodings: + result = self.ioclass() + gen = XMLGenerator(result, encoding=encoding) + + gen.startDocument() + gen.startElement("doc", {"a": u'\u20ac'}) + gen.characters(u"\u20ac".encode(encoding)) + gen.ignorableWhitespace(" ".encode(encoding)) + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), ( + u'\n' + u'\u20ac ' % encoding + ).encode(encoding, 'xmlcharrefreplace')) + def test_xmlgen_ns(self): result = self.ioclass() gen = XMLGenerator(result) diff --git a/Lib/xml/sax/saxutils.py b/Lib/xml/sax/saxutils.py index 3d81a8e1bf4..1abcd9a0c4a 100644 --- a/Lib/xml/sax/saxutils.py +++ b/Lib/xml/sax/saxutils.py @@ -180,10 +180,14 @@ def endElementNS(self, name, qname): self._write(u'' % self._qname(name)) def characters(self, content): - self._write(escape(unicode(content))) + if not isinstance(content, unicode): + content = unicode(content, self._encoding) + self._write(escape(content)) def ignorableWhitespace(self, content): - self._write(unicode(content)) + if not isinstance(content, unicode): + content = unicode(content, self._encoding) + self._write(content) def processingInstruction(self, target, data): self._write(u'' % (target, data)) diff --git a/Misc/ACKS b/Misc/ACKS index 7f78dbdeb7b..37ed4ce01e0 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1045,6 +1045,7 @@ Case Van Horsen Kyle VanderBeek Atul Varma Dmitry Vasiliev +Sebastian Ortiz Vasquez Alexandre Vassalotti Frank Vercruesse Mike Verdone diff --git a/Misc/NEWS b/Misc/NEWS index 16023468809..6cdf42b171b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -38,6 +38,10 @@ Core and Builtins Library ------- +- Issue #17606: Fixed support of encoded byte strings in the XMLGenerator + .characters() and ignorableWhitespace() methods. Original patch by Sebastian + Ortiz Vasquez. + - Issue #16601: Restarting iteration over tarfile no more continues from where it left off. Patch by Michael Birtwell. From 77ce915c4b5b5c55d9281526bc1a7a56367a892d Mon Sep 17 00:00:00 2001 From: doko Date: Wed, 15 May 2013 15:46:11 +0200 Subject: [PATCH 126/480] - Issue #17754: Make ctypes.util.find_library() independent of the locale. --- Lib/ctypes/util.py | 2 +- Misc/NEWS | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index fe2b5205325..fe0ed0a085f 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -93,7 +93,7 @@ def _findLib_gcc(name): fdout, ccout = tempfile.mkstemp() os.close(fdout) cmd = 'if type gcc >/dev/null 2>&1; then CC=gcc; elif type cc >/dev/null 2>&1; then CC=cc;else exit 10; fi;' \ - '$CC -Wl,-t -o ' + ccout + ' 2>&1 -l' + name + 'LANG=C LC_ALL=C $CC -Wl,-t -o ' + ccout + ' 2>&1 -l' + name try: f = os.popen(cmd) try: diff --git a/Misc/NEWS b/Misc/NEWS index 6cdf42b171b..baf06d0ca30 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,7 @@ Core and Builtins Library ------- +- Issue #17754: Make ctypes.util.find_library() independent of the locale. What's New in Python 2.7.5? =========================== From 23644ca29c2e27ccd01555e1da33812c9d25114f Mon Sep 17 00:00:00 2001 From: doko Date: Wed, 15 May 2013 16:54:16 +0200 Subject: [PATCH 127/480] - Fix typos in the multiprocessing module. --- Lib/multiprocessing/synchronize.py | 2 +- Misc/NEWS | 2 ++ Modules/_multiprocessing/multiprocessing.c | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 4b077e53aed..d845f72a1c4 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -226,7 +226,7 @@ def __repr__(self): num_waiters = (self._sleeping_count._semlock._get_value() - self._woken_count._semlock._get_value()) except Exception: - num_waiters = 'unkown' + num_waiters = 'unknown' return '' % (self._lock, num_waiters) def wait(self, timeout=None): diff --git a/Misc/NEWS b/Misc/NEWS index baf06d0ca30..c8dc24b11d8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -14,6 +14,8 @@ Library - Issue #17754: Make ctypes.util.find_library() independent of the locale. +- Fix typos in the multiprocessing module. + What's New in Python 2.7.5? =========================== diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index c831637f901..d192a074ba5 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -63,7 +63,7 @@ mp_SetError(PyObject *Type, int num) break; default: PyErr_Format(PyExc_RuntimeError, - "unkown error number %d", num); + "unknown error number %d", num); } return NULL; } From 644f1144ed821f0823e4e7f6dbabff42724c86f5 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Wed, 15 May 2013 18:00:45 -0700 Subject: [PATCH 128/480] Issue #17990: Only modify include and library search paths when cross-compiling. --- setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 716f08e2112..8d8ec84d814 100644 --- a/setup.py +++ b/setup.py @@ -437,9 +437,11 @@ def add_gcc_paths(self): def detect_modules(self): # Ensure that /usr/local is always used - add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib') - add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') - self.add_gcc_paths() + if not cross_compiling: + add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib') + add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') + if cross_compiling: + self.add_gcc_paths() self.add_multiarch_paths() # Add paths specified in the environment variables LDFLAGS and From c2b447957c8c4ba245b93aa515146ca04bcd7ce4 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 16 May 2013 22:47:47 +0100 Subject: [PATCH 129/480] Issue #17981: Closed socket on error in SysLogHandler. --- Lib/logging/handlers.py | 1 + Misc/NEWS | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 5837f4eda35..c4b8746ee5a 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -856,6 +856,7 @@ def emit(self, record): try: self.socket.send(msg) except socket.error: + self.socket.close() # See issue 17981 self._connect_unixsocket(self.address) self.socket.send(msg) elif self.socktype == socket.SOCK_DGRAM: diff --git a/Misc/NEWS b/Misc/NEWS index c8dc24b11d8..394b08b086b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,8 @@ Core and Builtins Library ------- +- Issue #17981: Closed socket on error in SysLogHandler. + - Issue #17754: Make ctypes.util.find_library() independent of the locale. - Fix typos in the multiprocessing module. From 3771977a7bafbf0fb824baad7072fea48b55f411 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 19 May 2013 15:44:54 +0200 Subject: [PATCH 130/480] Issue #11995: test_pydoc doesn't import all sys.path modules anymore. --- Lib/test/test_pydoc.py | 57 ++++++++++++++++++++++++++++++++++++------ Misc/NEWS | 6 +++++ 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 382d6dc878f..9e62f2859fe 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -4,15 +4,17 @@ import __builtin__ import re import pydoc +import contextlib import inspect import keyword +import pkgutil import unittest import xml.etree import test.test_support from collections import namedtuple from test.script_helper import assert_python_ok from test.test_support import ( - TESTFN, rmtree, reap_children, captured_stdout) + TESTFN, rmtree, reap_children, captured_stdout, captured_stderr) from test import pydoc_mod @@ -228,7 +230,30 @@ def print_diffs(text1, text2): print '\n' + ''.join(diffs) -class PyDocDocTest(unittest.TestCase): +class PydocBaseTest(unittest.TestCase): + + def _restricted_walk_packages(self, walk_packages, path=None): + """ + A version of pkgutil.walk_packages() that will restrict itself to + a given path. + """ + default_path = path or [os.path.dirname(__file__)] + def wrapper(path=None, prefix='', onerror=None): + return walk_packages(path or default_path, prefix, onerror) + return wrapper + + @contextlib.contextmanager + def restrict_walk_packages(self, path=None): + walk_packages = pkgutil.walk_packages + pkgutil.walk_packages = self._restricted_walk_packages(walk_packages, + path) + try: + yield + finally: + pkgutil.walk_packages = walk_packages + + +class PydocDocTest(unittest.TestCase): @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") @@ -303,7 +328,7 @@ def test_stripid(self): "") -class PydocImportTest(unittest.TestCase): +class PydocImportTest(PydocBaseTest): def setUp(self): self.test_dir = os.mkdir(TESTFN) @@ -338,8 +363,19 @@ def test_apropos_with_bad_package(self): badsyntax = os.path.join(pkgdir, "__init__") + os.extsep + "py" with open(badsyntax, 'w') as f: f.write("invalid python syntax = $1\n") - result = run_pydoc('zqwykjv', '-k', PYTHONPATH=TESTFN) - self.assertEqual('', result) + with self.restrict_walk_packages(path=[TESTFN]): + with captured_stdout() as out: + with captured_stderr() as err: + pydoc.apropos('xyzzy') + # No result, no error + self.assertEqual(out.getvalue(), '') + self.assertEqual(err.getvalue(), '') + # The package name is still matched + with captured_stdout() as out: + with captured_stderr() as err: + pydoc.apropos('syntaxerr') + self.assertEqual(out.getvalue().strip(), 'syntaxerr') + self.assertEqual(err.getvalue(), '') def test_apropos_with_unreadable_dir(self): # Issue 7367 - pydoc -k failed when unreadable dir on path @@ -348,8 +384,13 @@ def test_apropos_with_unreadable_dir(self): self.addCleanup(os.rmdir, self.unreadable_dir) # Note, on Windows the directory appears to be still # readable so this is not really testing the issue there - result = run_pydoc('zqwykjv', '-k', PYTHONPATH=TESTFN) - self.assertEqual('', result) + with self.restrict_walk_packages(path=[TESTFN]): + with captured_stdout() as out: + with captured_stderr() as err: + pydoc.apropos('SOMEKEY') + # No result, no error + self.assertEqual(out.getvalue(), '') + self.assertEqual(err.getvalue(), '') class TestDescriptions(unittest.TestCase): @@ -412,7 +453,7 @@ def test_builtin(self): def test_main(): try: - test.test_support.run_unittest(PyDocDocTest, + test.test_support.run_unittest(PydocDocTest, PydocImportTest, TestDescriptions, TestHelper) diff --git a/Misc/NEWS b/Misc/NEWS index 394b08b086b..efd7724a8e5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -18,6 +18,12 @@ Library - Fix typos in the multiprocessing module. +Tests +----- + +- Issue #11995: test_pydoc doesn't import all sys.path modules anymore. + + What's New in Python 2.7.5? =========================== From 4f6ca4f4ff1bbdd76a9c388ddb48b25f87b861ca Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 19 May 2013 19:38:12 -0700 Subject: [PATCH 131/480] add missing NULL check (closes #18019) --- Lib/test/test_dictviews.py | 5 +++++ Misc/NEWS | 3 +++ Objects/dictobject.c | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index f90367660c5..b264ff2ba16 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -144,6 +144,11 @@ def test_items_set_operations(self): self.assertEqual(d1.viewitems() ^ d3.viewitems(), {('a', 1), ('b', 2), ('d', 4), ('e', 5)}) + def test_recursive_repr(self): + d = {} + d[42] = d.viewvalues() + self.assertRaises(RuntimeError, repr, d) + diff --git a/Misc/NEWS b/Misc/NEWS index efd7724a8e5..cb7fd6006e3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -9,6 +9,9 @@ What's New in Python 2.7.6? Core and Builtins ----------------- +- Issue #18019: Fix crash in the repr of dictionaries containing their own + views. + Library ------- diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ba36b180cad..39e70359010 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2919,6 +2919,10 @@ dictview_repr(dictviewobject *dv) return NULL; seq_str = PyObject_Repr(seq); + if (seq_str == NULL) { + Py_DECREF(seq); + return NULL; + } result = PyString_FromFormat("%s(%s)", Py_TYPE(dv)->tp_name, PyString_AS_STRING(seq_str)); Py_DECREF(seq_str); From 4052886182e64431a7bf02fdbace419b00b11d51 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Mon, 20 May 2013 10:34:58 -0400 Subject: [PATCH 132/480] #17973: Add FAQ entry for ([],)[0] += [1] both extending and raising. This has come up often enough now on the tracker that it deserves a FAQ entry. --- Doc/faq/programming.rst | 83 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 2352db8e6ea..b351eca1417 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1223,6 +1223,89 @@ More generically, you can try the following function:: return map(apply, methods, [arguments]*nobjects) +Why does a_tuple[i] += ['item'] raise an exception when the addition works? +--------------------------------------------------------------------------- + +This is because of a combination of the fact that augmented assignment +operators are *assignment* operators, and the difference between mutable and +immutable objects in Python. + +This discussion applies in general when augmented assignment operators are +applied to elements of a tuple that point to mutable objects, but we'll use +a ``list`` and ``+=`` as our exemplar. + +If you wrote:: + + >>> a_tuple = (1, 2) + >>> a_tuple[0] += 1 + Traceback (most recent call last): + ... + TypeError: 'tuple' object does not support item assignment + +The reason for the exception should be immediately clear: ``1`` is added to the +object ``a_tuple[0]`` points to (``1``), producing the result object, ``2``, +but when we attempt to assign the result of the computation, ``2``, to element +``0`` of the tuple, we get an error because we can't change what an element of +a tuple points to. + +Under the covers, what this augmented assignment statement is doing is +approximately this:: + + >>> result = a_tuple[0].__iadd__(1) + >>> a_tuple[0] = result + Traceback (most recent call last): + ... + TypeError: 'tuple' object does not support item assignment + +It is the assignment part of the operation that produces the error, since a +tuple is immutable. + +When you write something like:: + + >>> a_tuple = (['foo'], 'bar') + >>> a_tuple[0] += ['item'] + Traceback (most recent call last): + ... + TypeError: 'tuple' object does not support item assignment + +The exception is a bit more surprising, and even more surprising is the fact +that even though there was an error, the append worked:: + + >>> a_tuple[0] + ['foo', 'item'] + +To see why this happens, you need to know that for lists, ``__iadd__`` is equivalent +to calling ``extend`` on the list and returning the list. That's why we say +that for lists, ``+=`` is a "shorthand" for ``list.extend``:: + + >>> a_list = [] + >>> a_list += [1] + >>> a_list + [1] + +is equivalent to:: + + >>> result = a_list.__iadd__([1]) + >>> a_list = result + +The object pointed to by a_list has been mutated, and the pointer to the +mutated object is assigned back to ``a_list``. The end result of the +assignment is a no-op, since it is a pointer to the same object that ``a_list`` +was previously pointing to, but the assignment still happens. + +Thus, in our tuple example what is happening is equivalent to:: + + >>> result = a_tuple[0].__iadd__(['item']) + >>> a_tuple[0] = result + Traceback (most recent call last): + ... + TypeError: 'tuple' object does not support item assignment + +The ``__iadd__`` succeeds, and thus the list is extended, but even though +``result`` points to the same object that ``a_tuple[0]`` already points to, +that final assignment still results in an error, because tuples are immutable. + + Dictionaries ============ From cc1b3c99414c5991fb631aca2bdda7e126cc3616 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 20 May 2013 14:27:06 -0700 Subject: [PATCH 133/480] Issue #18026: fix ctypes doc typo --- Doc/library/ctypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index c72a0b5a4ed..6be8debaf02 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1333,7 +1333,7 @@ returns the full pathname, but since there is no predefined naming scheme a call like ``find_library("c")`` will fail and return ``None``. If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine -the shared library name at development type, and hardcode that into the wrapper +the shared library name at development time, and hardcode that into the wrapper module instead of using :func:`find_library` to locate the library at runtime. From 6a6f55544c7d2a0a8b84e9eae652d24a13203920 Mon Sep 17 00:00:00 2001 From: Roger Serwy Date: Mon, 20 May 2013 22:13:39 -0500 Subject: [PATCH 134/480] #14146: Highlight source line while debugging on Windows. --- Lib/idlelib/EditorWindow.py | 30 ++++++++++++++++++++++++++++++ Misc/NEWS | 6 ++++++ 2 files changed, 36 insertions(+) diff --git a/Lib/idlelib/EditorWindow.py b/Lib/idlelib/EditorWindow.py index cf3fad3c1ec..38cbfcfc559 100644 --- a/Lib/idlelib/EditorWindow.py +++ b/Lib/idlelib/EditorWindow.py @@ -346,6 +346,36 @@ def __init__(self, flist=None, filename=None, key=None, root=None): self.askinteger = tkSimpleDialog.askinteger self.showerror = tkMessageBox.showerror + self._highlight_workaround() # Fix selection tags on Windows + + def _highlight_workaround(self): + # On Windows, Tk removes painting of the selection + # tags which is different behavior than on Linux and Mac. + # See issue14146 for more information. + if not sys.platform.startswith('win'): + return + + text = self.text + text.event_add("<>", "") + text.event_add("<>", "") + def highlight_fix(focus): + sel_range = text.tag_ranges("sel") + if sel_range: + if focus == 'out': + HILITE_CONFIG = idleConf.GetHighlight( + idleConf.CurrentTheme(), 'hilite') + text.tag_config("sel_fix", HILITE_CONFIG) + text.tag_raise("sel_fix") + text.tag_add("sel_fix", *sel_range) + elif focus == 'in': + text.tag_remove("sel_fix", "1.0", "end") + + text.bind("<>", + lambda ev: highlight_fix("out")) + text.bind("<>", + lambda ev: highlight_fix("in")) + + def _filename_to_unicode(self, filename): """convert filename to unicode in order to display it in Tk""" if isinstance(filename, unicode) or not filename: diff --git a/Misc/NEWS b/Misc/NEWS index cb7fd6006e3..b2485eff3e6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,12 @@ Library - Fix typos in the multiprocessing module. +IDLE +---- + +- Issue #14146: Highlight source line while debugging on Windows. + + Tests ----- From a09aa913e70d8f186bc45694195cdae3c31f689e Mon Sep 17 00:00:00 2001 From: R David Murray Date: Tue, 21 May 2013 11:46:18 -0400 Subject: [PATCH 135/480] #17973: fix technical inaccuracy in faq entry (it now passes doctest). --- Doc/faq/programming.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index b351eca1417..b9ed53410c7 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1251,7 +1251,7 @@ a tuple points to. Under the covers, what this augmented assignment statement is doing is approximately this:: - >>> result = a_tuple[0].__iadd__(1) + >>> result = a_tuple[0] + 1 >>> a_tuple[0] = result Traceback (most recent call last): ... @@ -1274,16 +1274,19 @@ that even though there was an error, the append worked:: >>> a_tuple[0] ['foo', 'item'] -To see why this happens, you need to know that for lists, ``__iadd__`` is equivalent -to calling ``extend`` on the list and returning the list. That's why we say -that for lists, ``+=`` is a "shorthand" for ``list.extend``:: +To see why this happens, you need to know that (a) if an object implements an +``__iadd__`` magic method, it gets called when the ``+=`` augmented assignment +is executed, and its return value is what gets used in the assignment statement; +and (b) for lists, ``__iadd__`` is equivalent to calling ``extend`` on the list +and returning the list. That's why we say that for lists, ``+=`` is a +"shorthand" for ``list.extend``:: >>> a_list = [] >>> a_list += [1] >>> a_list [1] -is equivalent to:: +This is equivalent to:: >>> result = a_list.__iadd__([1]) >>> a_list = result From d65704ae530c05bd343b4fa938e460bd2f71449a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 21 May 2013 22:53:33 +0300 Subject: [PATCH 136/480] Issue #17979: Fixed the re module in build with --disable-unicode. --- Misc/NEWS | 2 ++ Modules/sre.h | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index b2485eff3e6..9367bd54f53 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -58,6 +58,8 @@ Core and Builtins Library ------- +- Issue #17979: Fixed the re module in build with --disable-unicode. + - Issue #17606: Fixed support of encoded byte strings in the XMLGenerator .characters() and ignorableWhitespace() methods. Original patch by Sebastian Ortiz Vasquez. diff --git a/Modules/sre.h b/Modules/sre.h index 200e492d264..15ed3d511b7 100644 --- a/Modules/sre.h +++ b/Modules/sre.h @@ -23,8 +23,8 @@ # define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX + 1u) # endif #else -# define SRE_CODE unsigned long -# if SIZEOF_SIZE_T > SIZEOF_LONG +# define SRE_CODE unsigned int +# if SIZEOF_SIZE_T > SIZEOF_INT # define SRE_MAXREPEAT (~(SRE_CODE)0) # else # define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX + 1u) From f642e1b3fe702c9081743a6775be258055062ffc Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 22 May 2013 15:28:30 +0300 Subject: [PATCH 137/480] Issue #17844: Refactor a documentation of Python specific encodings. Add links to encoders and decoders for binary-to-binary codecs. --- Doc/library/codecs.rst | 180 +++++++++++++++++++++++------------------ Misc/NEWS | 7 +- 2 files changed, 108 insertions(+), 79 deletions(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 460af13d33e..39a3c5dff60 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1098,88 +1098,112 @@ particular, the following variants typically exist: | utf_8_sig | | all languages | +-----------------+--------------------------------+--------------------------------+ -A number of codecs are specific to Python, so their codec names have no meaning -outside Python. Some of them don't convert from Unicode strings to byte strings, -but instead use the property of the Python codecs machinery that any bijective -function with one argument can be considered as an encoding. - -For the codecs listed below, the result in the "encoding" direction is always a -byte string. The result of the "decoding" direction is listed as operand type in -the table. - -.. tabularcolumns:: |l|p{0.3\linewidth}|l|p{0.3\linewidth}| - -+--------------------+---------------------------+----------------+---------------------------+ -| Codec | Aliases | Operand type | Purpose | -+====================+===========================+================+===========================+ -| base64_codec | base64, base-64 | byte string | Convert operand to MIME | -| | | | base64 (the result always | -| | | | includes a trailing | -| | | | ``'\n'``) | -+--------------------+---------------------------+----------------+---------------------------+ -| bz2_codec | bz2 | byte string | Compress the operand | -| | | | using bz2 | -+--------------------+---------------------------+----------------+---------------------------+ -| hex_codec | hex | byte string | Convert operand to | -| | | | hexadecimal | -| | | | representation, with two | -| | | | digits per byte | -+--------------------+---------------------------+----------------+---------------------------+ -| idna | | Unicode string | Implements :rfc:`3490`, | -| | | | see also | -| | | | :mod:`encodings.idna` | -+--------------------+---------------------------+----------------+---------------------------+ -| mbcs | dbcs | Unicode string | Windows only: Encode | -| | | | operand according to the | -| | | | ANSI codepage (CP_ACP) | -+--------------------+---------------------------+----------------+---------------------------+ -| palmos | | Unicode string | Encoding of PalmOS 3.5 | -+--------------------+---------------------------+----------------+---------------------------+ -| punycode | | Unicode string | Implements :rfc:`3492` | -+--------------------+---------------------------+----------------+---------------------------+ -| quopri_codec | quopri, quoted-printable, | byte string | Convert operand to MIME | -| | quotedprintable | | quoted printable | -+--------------------+---------------------------+----------------+---------------------------+ -| raw_unicode_escape | | Unicode string | Produce a string that is | -| | | | suitable as raw Unicode | -| | | | literal in Python source | -| | | | code | -+--------------------+---------------------------+----------------+---------------------------+ -| rot_13 | rot13 | Unicode string | Returns the Caesar-cypher | -| | | | encryption of the operand | -+--------------------+---------------------------+----------------+---------------------------+ -| string_escape | | byte string | Produce a string that is | -| | | | suitable as string | -| | | | literal in Python source | -| | | | code | -+--------------------+---------------------------+----------------+---------------------------+ -| undefined | | any | Raise an exception for | -| | | | all conversions. Can be | -| | | | used as the system | -| | | | encoding if no automatic | -| | | | :term:`coercion` between | -| | | | byte and Unicode strings | -| | | | is desired. | -+--------------------+---------------------------+----------------+---------------------------+ -| unicode_escape | | Unicode string | Produce a string that is | -| | | | suitable as Unicode | -| | | | literal in Python source | -| | | | code | -+--------------------+---------------------------+----------------+---------------------------+ -| unicode_internal | | Unicode string | Return the internal | -| | | | representation of the | -| | | | operand | -+--------------------+---------------------------+----------------+---------------------------+ -| uu_codec | uu | byte string | Convert the operand using | -| | | | uuencode | -+--------------------+---------------------------+----------------+---------------------------+ -| zlib_codec | zip, zlib | byte string | Compress the operand | -| | | | using gzip | -+--------------------+---------------------------+----------------+---------------------------+ +Python Specific Encodings +------------------------- + +A number of predefined codecs are specific to Python, so their codec names have +no meaning outside Python. These are listed in the tables below based on the +expected input and output types (note that while text encodings are the most +common use case for codecs, the underlying codec infrastructure supports +arbitrary data transforms rather than just text encodings). For asymmetric +codecs, the stated purpose describes the encoding direction. + +The following codecs provide unicode-to-str encoding [#encoding-note]_ and +str-to-unicode decoding [#decoding-note]_, similar to the Unicode text +encodings. + +.. tabularcolumns:: |l|L|L| + ++--------------------+---------------------------+---------------------------+ +| Codec | Aliases | Purpose | ++====================+===========================+===========================+ +| idna | | Implements :rfc:`3490`, | +| | | see also | +| | | :mod:`encodings.idna` | ++--------------------+---------------------------+---------------------------+ +| mbcs | dbcs | Windows only: Encode | +| | | operand according to the | +| | | ANSI codepage (CP_ACP) | ++--------------------+---------------------------+---------------------------+ +| palmos | | Encoding of PalmOS 3.5 | ++--------------------+---------------------------+---------------------------+ +| punycode | | Implements :rfc:`3492` | ++--------------------+---------------------------+---------------------------+ +| raw_unicode_escape | | Produce a string that is | +| | | suitable as raw Unicode | +| | | literal in Python source | +| | | code | ++--------------------+---------------------------+---------------------------+ +| rot_13 | rot13 | Returns the Caesar-cypher | +| | | encryption of the operand | ++--------------------+---------------------------+---------------------------+ +| undefined | | Raise an exception for | +| | | all conversions. Can be | +| | | used as the system | +| | | encoding if no automatic | +| | | :term:`coercion` between | +| | | byte and Unicode strings | +| | | is desired. | ++--------------------+---------------------------+---------------------------+ +| unicode_escape | | Produce a string that is | +| | | suitable as Unicode | +| | | literal in Python source | +| | | code | ++--------------------+---------------------------+---------------------------+ +| unicode_internal | | Return the internal | +| | | representation of the | +| | | operand | ++--------------------+---------------------------+---------------------------+ .. versionadded:: 2.3 The ``idna`` and ``punycode`` encodings. +The following codecs provide str-to-str encoding and decoding +[#decoding-note]_. + +.. tabularcolumns:: |l|L|L|L| + ++--------------------+---------------------------+---------------------------+------------------------------+ +| Codec | Aliases | Purpose | Encoder/decoder | ++====================+===========================+===========================+==============================+ +| base64_codec | base64, base-64 | Convert operand to MIME | :meth:`base64.b64encode`, | +| | | base64 (the result always | :meth:`base64.b64decode` | +| | | includes a trailing | | +| | | ``'\n'``) | | ++--------------------+---------------------------+---------------------------+------------------------------+ +| bz2_codec | bz2 | Compress the operand | :meth:`bz2.compress`, | +| | | using bz2 | :meth:`bz2.decompress` | ++--------------------+---------------------------+---------------------------+------------------------------+ +| hex_codec | hex | Convert operand to | :meth:`base64.b16encode`, | +| | | hexadecimal | :meth:`base64.b16decode` | +| | | representation, with two | | +| | | digits per byte | | ++--------------------+---------------------------+---------------------------+------------------------------+ +| quopri_codec | quopri, quoted-printable, | Convert operand to MIME | :meth:`quopri.encodestring`, | +| | quotedprintable | quoted printable | :meth:`quopri.decodestring` | ++--------------------+---------------------------+---------------------------+------------------------------+ +| string_escape | | Produce a string that is | | +| | | suitable as string | | +| | | literal in Python source | | +| | | code | | ++--------------------+---------------------------+---------------------------+------------------------------+ +| uu_codec | uu | Convert the operand using | :meth:`uu.encode`, | +| | | uuencode | :meth:`uu.decode` | ++--------------------+---------------------------+---------------------------+------------------------------+ +| zlib_codec | zip, zlib | Compress the operand | :meth:`zlib.compress`, | +| | | using gzip | :meth:`zlib.decompress` | ++--------------------+---------------------------+---------------------------+------------------------------+ + +.. [#encoding-note] str objects are also accepted as input in place of unicode + objects. They are implicitly converted to unicode by decoding them using + the default encoding. If this conversion fails, it may lead to encoding + operations raising :exc:`UnicodeDecodeError`. + +.. [#decoding-note] unicode objects are also accepted as input in place of str + objects. They are implicitly converted to str by encoding them using the + default encoding. If this conversion fails, it may lead to decoding + operations raising :exc:`UnicodeEncodeError`. + :mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------ diff --git a/Misc/NEWS b/Misc/NEWS index 9367bd54f53..391ecc5288c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -26,12 +26,17 @@ IDLE - Issue #14146: Highlight source line while debugging on Windows. - Tests ----- - Issue #11995: test_pydoc doesn't import all sys.path modules anymore. +Documentation +------------- + +- Issue #17844: Refactor a documentation of Python specific encodings. + Add links to encoders and decoders for binary-to-binary codecs. + What's New in Python 2.7.5? =========================== From 3db6b9dc2875357b552328250ed56a8dd8037230 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 22 May 2013 13:25:41 -0700 Subject: [PATCH 138/480] implement missing inequality on WeakSet --- Lib/_weakrefset.py | 6 ++++++ Lib/test/test_weakset.py | 6 ++++++ Misc/NEWS | 2 ++ 3 files changed, 14 insertions(+) diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index 43f9a6e447d..990c3a6bbcd 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -171,6 +171,12 @@ def __eq__(self, other): return NotImplemented return self.data == set(ref(item) for item in other) + def __ne__(self, other): + opposite = self.__eq__(other) + if opposite is NotImplemented: + return NotImplemented + return not opposite + def symmetric_difference(self, other): newset = self.copy() newset.symmetric_difference_update(other) diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index d9e091ed02f..fb9e8d73db3 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -351,6 +351,12 @@ def test_eq(self): self.assertFalse(self.s == tuple(self.items)) self.assertFalse(self.s == 1) + def test_ne(self): + self.assertTrue(self.s != set(self.items)) + s1 = WeakSet() + s2 = WeakSet() + self.assertFalse(s1 != s2) + def test_weak_destroy_while_iterating(self): # Issue #7105: iterators shouldn't crash when a key is implicitly removed # Create new items to be sure no-one else holds a reference diff --git a/Misc/NEWS b/Misc/NEWS index 391ecc5288c..7a720309184 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -15,6 +15,8 @@ Core and Builtins Library ------- +- Implement inequality on weakref.WeakSet. + - Issue #17981: Closed socket on error in SysLogHandler. - Issue #17754: Make ctypes.util.find_library() independent of the locale. From 7088a61d4b77567d24cfc390dad94a087ce4084f Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Wed, 22 May 2013 15:16:17 -0700 Subject: [PATCH 139/480] =?UTF-8?q?Issue=20#17532:=20Always=20include=20Op?= =?UTF-8?q?tions=20menu=20for=20IDLE=20on=20OS=20X.=20Patch=20by=20Guilher?= =?UTF-8?q?me=20Sim=C3=B5es.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/idlelib/Bindings.py | 4 ++++ Lib/idlelib/EditorWindow.py | 1 - Lib/idlelib/PyShell.py | 1 - Misc/ACKS | 1 + Misc/NEWS | 3 +++ 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/Bindings.py b/Lib/idlelib/Bindings.py index ec2720b0d02..fe6e812d2bf 100644 --- a/Lib/idlelib/Bindings.py +++ b/Lib/idlelib/Bindings.py @@ -98,6 +98,10 @@ # menu del menudefs[-1][1][0:2] + # Remove the 'Configure' entry from the options menu, it is in the + # application menu as 'Preferences' + del menudefs[-2][1][0:2] + default_keydefs = idleConf.GetCurrentKeySet() del sys diff --git a/Lib/idlelib/EditorWindow.py b/Lib/idlelib/EditorWindow.py index 38cbfcfc559..7b9b583db9a 100644 --- a/Lib/idlelib/EditorWindow.py +++ b/Lib/idlelib/EditorWindow.py @@ -467,7 +467,6 @@ def set_line_and_column(self, event=None): ] if macosxSupport.runningAsOSXApp(): - del menu_specs[-3] menu_specs[-2] = ("windows", "_Window") diff --git a/Lib/idlelib/PyShell.py b/Lib/idlelib/PyShell.py index 81af85a8404..0a77bb58536 100644 --- a/Lib/idlelib/PyShell.py +++ b/Lib/idlelib/PyShell.py @@ -844,7 +844,6 @@ class PyShell(OutputWindow): ] if macosxSupport.runningAsOSXApp(): - del menu_specs[-3] menu_specs[-2] = ("windows", "_Window") diff --git a/Misc/ACKS b/Misc/ACKS index 37ed4ce01e0..21a1a49381a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -941,6 +941,7 @@ Michael Simcich Ionel Simionescu Kirill Simonov Nathan Paul Simons +Guilherme Simões Ravi Sinha Janne Sinkkonen Ng Pheng Siong diff --git a/Misc/NEWS b/Misc/NEWS index 7a720309184..9c472ddbf9f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,6 +28,9 @@ IDLE - Issue #14146: Highlight source line while debugging on Windows. +- Issue #17532: Always include Options menu for IDLE on OS X. + Patch by Guilherme Simões. + Tests ----- From b60be31714dc7595086fe2cb30510617b45488e2 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 23 May 2013 00:12:14 -0700 Subject: [PATCH 140/480] Issue #18031: %-formatting isn't dead yet and might pull through. --- Doc/tutorial/inputoutput.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index b1611f20590..45ca79244c4 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -215,10 +215,6 @@ operation. For example:: >>> print 'The value of PI is approximately %5.3f.' % math.pi The value of PI is approximately 3.142. -Since :meth:`str.format` is quite new, a lot of Python code still uses the ``%`` -operator. However, because this old style of formatting will eventually be -removed from the language, :meth:`str.format` should generally be used. - More information can be found in the :ref:`string-formatting` section. From 7c530239d2c701c6e80df0ffee692b5f64e05672 Mon Sep 17 00:00:00 2001 From: David Wolever Date: Thu, 23 May 2013 17:23:49 -0400 Subject: [PATCH 141/480] Remove Japanese datetime examples --- Doc/library/datetime.rst | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index cf41d434c0e..ad1e0474ec1 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1619,16 +1619,12 @@ platforms. Regardless of platform, years before 1900 cannot be used. | ``%a`` | Weekday as locale's || Sun, Mon, ..., Sat | \(1) | | | abbreviated name. | (en_US); | | | | || So, Mo, ..., Sa | | -| | | (de_DE); | | -| | || æ—¥, 月, ..., 土 | | -| | | (ja_JP) | | +| | | (de_DE) | | +-----------+--------------------------------+------------------------+-------+ | ``%A`` | Weekday as locale's full name. || Sunday, Monday, ..., | \(1) | | | | Saturday (en_US); | | | | || Sonntag, Montag, ..., | | -| | | Samstag (de_DE); | | -| | || 日曜日, 月曜日, ..., | | -| | | 土曜日 (ja_JP) | | +| | | Samstag (de_DE) | | +-----------+--------------------------------+------------------------+-------+ | ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | | | where 0 is Sunday and 6 is | | | @@ -1640,15 +1636,12 @@ platforms. Regardless of platform, years before 1900 cannot be used. | ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | | | name. | (en_US); | | | | || Jan, Feb, ..., Dez | | -| | | (de_DE); | | -| | || 1, 2, ..., 12 (ja_JP) | | +| | | (de_DE) | | +-----------+--------------------------------+------------------------+-------+ | ``%B`` | Month as locale's full name. || January, February, | \(1) | | | | ..., December (en_US);| | | | || Januar, Februar, ..., | | -| | | Dezember (de_DE); | | -| | || 1月, 2月, ..., 12月 | | -| | | (ja_JP) | | +| | | Dezember (de_DE) | | +-----------+--------------------------------+------------------------+-------+ | ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | | | | decimal number. | | | @@ -1666,8 +1659,7 @@ platforms. Regardless of platform, years before 1900 cannot be used. | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | -| | AM or PM. || am, pm (de_DE); | \(2) | -| | || AM, PM (ja_JP) | | +| | AM or PM. || am, pm (de_DE) | \(2) | +-----------+--------------------------------+------------------------+-------+ | ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | | | | decimal number. | | | @@ -1706,20 +1698,16 @@ platforms. Regardless of platform, years before 1900 cannot be used. | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ | ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | -| | time representation. | 1988 (en_US), | | +| | time representation. | 1988 (en_US); | | | | || Di 16 Aug 21:30:00 | | -| | | 1988 (de_DE), | | -| | || ç« 8/16 21:30:00 | | -| | | 1988 (ja_JP) | | +| | | 1988 (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%x`` | Locale's appropriate date || 08/16/88 (None), | \(1) | -| | representation. || 08/16/1988 (en_US), | | -| | || 16.08.1988 (de_DE), | | -| | || 1988/08/16 (ja_JP) | | +| ``%x`` | Locale's appropriate date || 08/16/88 (None); | \(1) | +| | representation. || 08/16/1988 (en_US); | | +| | || 16.08.1988 (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%X`` | Locale's appropriate time || 21:30:00 (en_US), | \(1) | -| | representation. || 21:30:00 (de_DE), | | -| | || 21時30分00秒 (ja_JP) | | +| ``%X`` | Locale's appropriate time || 21:30:00 (en_US); | \(1) | +| | representation. || 21:30:00 (de_DE) | | +-----------+--------------------------------+------------------------+-------+ | ``%%`` | A literal ``'%'`` character. | % | | +-----------+--------------------------------+------------------------+-------+ From 8253020b9d6032acd5c9357b9dd08df08f850ab5 Mon Sep 17 00:00:00 2001 From: David Wolever Date: Thu, 23 May 2013 17:42:14 -0400 Subject: [PATCH 142/480] Clarify that strftime's encoding is based on locale --- Doc/library/datetime.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index ad1e0474ec1..e33f3f06ba1 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1718,10 +1718,10 @@ Notes: Because the format depends on the current locale, care should be taken when making assumptions about the output value. Field orderings will vary (for example, "month/day/year" versus "day/month/year"), and the output may - contain Unicode characters (encoded, by default, using UTF-8, but this may - vary based on the locale; for example, the ``ja_JP`` locale contains UTF-8 - encoded Japanese characters, but ``ja_JP.SJIS`` contains Shift JIS encoded - Japanese characters). + contain Unicode characters encoded using the locale's default encoding (for + example, if the current locale is ``js_JP``, the default encoding could be + any one of ``eucJP``, ``SJIS``, or ``utf-8``; use :meth:`locale.getlocale` + to determine the current locale's encoding). (2) When used with the :meth:`strptime` method, the ``%p`` directive only affects From 4aab20bad3980fbaba8bf91a6c9bb58230e69735 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Fri, 24 May 2013 13:45:27 +0200 Subject: [PATCH 143/480] Issue #17269: Workaround for a platform bug in getaddrinfo on OSX Without this patch socket.getaddrinfo crashed when called with some unusual argument combinations. --- Lib/test/test_socket.py | 2 ++ Misc/NEWS | 5 ++++- Modules/socketmodule.c | 9 +++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 111e55394a6..3b112e68384 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -665,6 +665,8 @@ def testGetaddrinfo(self): socket.getaddrinfo(None, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE) + # Issue 17269 + socket.getaddrinfo("localhost", None, 0, 0, 0, socket.AI_NUMERICSERV) def check_sendall_interrupted(self, with_timeout): # socketpair() is not stricly required, but it makes things easier. diff --git a/Misc/NEWS b/Misc/NEWS index 9c472ddbf9f..8fdc8810a80 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -23,6 +23,9 @@ Library - Fix typos in the multiprocessing module. +- Issue #17269: Workaround for socket.getaddrinfo crash on MacOS X + with port None or "0" and flags AI_NUMERICSERV. + IDLE ---- @@ -51,7 +54,7 @@ What's New in Python 2.7.5? Core and Builtins ----------------- -- Issue #15535: Fixed regression in the pickling of named tuples by +- Issue #15535: Fixed regression in the pickling of named tuples by removing the __dict__ property introduced in 2.7.4. - Issue #17857: Prevent build failures with pre-3.5.0 versions of sqlite3, diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index bdc055dd384..238c849b845 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -4179,6 +4179,15 @@ socket_getaddrinfo(PyObject *self, PyObject *args) "getaddrinfo() argument 2 must be integer or string"); goto err; } +#ifdef __APPLE__ + if ((flags & AI_NUMERICSERV) && (pptr == NULL || (pptr[0] == '0' && pptr[1] == 0))) { + /* On OSX upto at least OSX 10.8 getaddrinfo crashes + * if AI_NUMERICSERV is set and the servname is NULL or "0". + * This workaround avoids a segfault in libsystem. + */ + pptr = "00"; + } +#endif memset(&hints, 0, sizeof(hints)); hints.ai_family = family; hints.ai_socktype = socktype; From b080e95cc5b9111b4ef04942d578125275137ba2 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 27 May 2013 10:58:55 -0700 Subject: [PATCH 144/480] Issue #18015: Fix unpickling of 2.7.3 and 2.7.4 namedtuples. --- Doc/library/collections.rst | 6 +++++ Lib/collections.py | 6 +++++ Lib/test/test_collections.py | 44 +++++++++++++++++++++++++++++++++++- Misc/NEWS | 2 ++ 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index ab290e79f35..b7628bb8deb 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -638,6 +638,12 @@ Example: def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) + + __dict__ = _property(_asdict) + + def __getstate__(self): + 'Exclude the OrderedDict from pickling' + pass x = _property(_itemgetter(0), doc='Alias for field number 0') diff --git a/Lib/collections.py b/Lib/collections.py index af32e448a9c..b9df2c14c91 100644 --- a/Lib/collections.py +++ b/Lib/collections.py @@ -270,6 +270,12 @@ def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) + __dict__ = _property(_asdict) + + def __getstate__(self): + 'Exclude the OrderedDict from pickling' + pass + {field_defs} ''' diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 8bdeb3d8f1f..3aaecb2a6f9 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -17,6 +17,43 @@ TestNT = namedtuple('TestNT', 'x y z') # type used for pickle tests +py273_named_tuple_pickle = '''\ +ccopy_reg +_reconstructor +p0 +(ctest.test_collections +TestNT +p1 +c__builtin__ +tuple +p2 +(I10 +I20 +I30 +tp3 +tp4 +Rp5 +ccollections +OrderedDict +p6 +((lp7 +(lp8 +S'x' +p9 +aI10 +aa(lp10 +S'y' +p11 +aI20 +aa(lp12 +S'z' +p13 +aI30 +aatp14 +Rp15 +b. +''' + class TestNamedTuple(unittest.TestCase): def test_factory(self): @@ -78,12 +115,12 @@ def test_instance(self): self.assertRaises(TypeError, eval, 'Point(XXX=1, y=2)', locals()) # wrong keyword argument self.assertRaises(TypeError, eval, 'Point(x=1)', locals()) # missing keyword argument self.assertEqual(repr(p), 'Point(x=11, y=22)') - self.assertNotIn('__dict__', dir(p)) # verify instance has no dict self.assertNotIn('__weakref__', dir(p)) self.assertEqual(p, Point._make([11, 22])) # test _make classmethod self.assertEqual(p._fields, ('x', 'y')) # test _fields attribute self.assertEqual(p._replace(x=1), (1, 22)) # test _replace method self.assertEqual(p._asdict(), dict(x=11, y=22)) # test _asdict method + self.assertEqual(vars(p), p._asdict()) # verify that vars() works try: p._replace(x=1, error=2) @@ -215,6 +252,11 @@ def test_name_conflicts(self): # test __getnewargs__ self.assertEqual(t.__getnewargs__(), values) + def test_pickling_bug_18015(self): + # http://bugs.python.org/issue18015 + pt = pickle.loads(py273_named_tuple_pickle) + self.assertEqual(pt.x, 10) + class ABCTestCase(unittest.TestCase): def validate_abstract_methods(self, abc, *names): diff --git a/Misc/NEWS b/Misc/NEWS index 8fdc8810a80..13f7cc360a3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -19,6 +19,8 @@ Library - Issue #17981: Closed socket on error in SysLogHandler. +- Issue #18015: Fix unpickling of 2.7.3 and 2.7.4 namedtuples. + - Issue #17754: Make ctypes.util.find_library() independent of the locale. - Fix typos in the multiprocessing module. From e2a20b4c363dd673affe932af836149b25422c83 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 28 May 2013 22:46:51 +0300 Subject: [PATCH 145/480] Issue #18085: Fix PyObject_CallMethodObjArgs()'s entry in refcounts.dat. --- Doc/data/refcounts.dat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 1fc896fb73b..06c19d0412a 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -932,7 +932,7 @@ PyObject_CallMethod::...:: PyObject_CallMethodObjArgs:PyObject*::+1: PyObject_CallMethodObjArgs:PyObject*:o:0: -PyObject_CallMethodObjArgs:char*:name:: +PyObject_CallMethodObjArgs:PyObject*:name:0: PyObject_CallMethodObjArgs::...:: PyObject_CallObject:PyObject*::+1: From 048a05eeed32284af783d6ad2b142548e5c705a6 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 28 May 2013 16:31:45 -0700 Subject: [PATCH 146/480] Issue #18080: When building a C extension module on OS X, if the compiler is overriden with the CC environment variable, use the new compiler as the default for linking if LDSHARED is not also overriden. This restores Distutils behavior introduced in 2.7.3 and inadvertently dropped in 2.7.4. --- Lib/distutils/sysconfig.py | 10 +++++-- Lib/distutils/tests/test_unixccompiler.py | 34 ++++++++++++++++++++++- Misc/NEWS | 5 ++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 0c726d92ffb..4aa93346d53 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -175,9 +175,15 @@ def customize_compiler(compiler): 'CCSHARED', 'LDSHARED', 'SO', 'AR', 'ARFLAGS') - newcc = None if 'CC' in os.environ: - cc = os.environ['CC'] + newcc = os.environ['CC'] + if (sys.platform == 'darwin' + and 'LDSHARED' not in os.environ + and ldshared.startswith(cc)): + # On OS X, if CC is overridden, use that as the default + # command for LDSHARED as well + ldshared = newcc + ldshared[len(cc):] + cc = newcc if 'CXX' in os.environ: cxx = os.environ['CXX'] if 'LDSHARED' in os.environ: diff --git a/Lib/distutils/tests/test_unixccompiler.py b/Lib/distutils/tests/test_unixccompiler.py index 40c908a24d0..64f9d267511 100644 --- a/Lib/distutils/tests/test_unixccompiler.py +++ b/Lib/distutils/tests/test_unixccompiler.py @@ -1,7 +1,8 @@ """Tests for distutils.unixccompiler.""" +import os import sys import unittest -from test.test_support import run_unittest +from test.test_support import EnvironmentVarGuard, run_unittest from distutils import sysconfig from distutils.unixccompiler import UnixCCompiler @@ -122,6 +123,37 @@ def gcv(v): sysconfig.get_config_var = gcv self.assertEqual(self.cc.rpath_foo(), '-R/foo') + @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for OS X') + def test_osx_cc_overrides_ldshared(self): + # Issue #18080: + # ensure that setting CC env variable also changes default linker + def gcv(v): + if v == 'LDSHARED': + return 'gcc-4.2 -bundle -undefined dynamic_lookup ' + return 'gcc-4.2' + sysconfig.get_config_var = gcv + with EnvironmentVarGuard() as env: + env['CC'] = 'my_cc' + del env['LDSHARED'] + sysconfig.customize_compiler(self.cc) + self.assertEqual(self.cc.linker_so[0], 'my_cc') + + @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for OS X') + def test_osx_explict_ldshared(self): + # Issue #18080: + # ensure that setting CC env variable does not change + # explicit LDSHARED setting for linker + def gcv(v): + if v == 'LDSHARED': + return 'gcc-4.2 -bundle -undefined dynamic_lookup ' + return 'gcc-4.2' + sysconfig.get_config_var = gcv + with EnvironmentVarGuard() as env: + env['CC'] = 'my_cc' + env['LDSHARED'] = 'my_ld -bundle -dynamic' + sysconfig.customize_compiler(self.cc) + self.assertEqual(self.cc.linker_so[0], 'my_ld') + def test_suite(): return unittest.makeSuite(UnixCCompilerTestCase) diff --git a/Misc/NEWS b/Misc/NEWS index 13f7cc360a3..00448c45af4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,6 +28,11 @@ Library - Issue #17269: Workaround for socket.getaddrinfo crash on MacOS X with port None or "0" and flags AI_NUMERICSERV. +- Issue #18080: When building a C extension module on OS X, if the compiler + is overriden with the CC environment variable, use the new compiler as + the default for linking if LDSHARED is not also overriden. This restores + Distutils behavior introduced in 2.7.3 and inadvertently dropped in 2.7.4. + IDLE ---- From aac6957f317fdc639455e8c17d2e9089d5f9b268 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 29 May 2013 05:58:47 -0700 Subject: [PATCH 147/480] #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline. This helps in handling certain types invalid urls in a conservative manner. --- Lib/robotparser.py | 1 + Lib/test/test_robotparser.py | 12 ++++++++++++ Misc/NEWS | 4 ++++ 3 files changed, 17 insertions(+) diff --git a/Lib/robotparser.py b/Lib/robotparser.py index 1722863d144..ad3be947115 100644 --- a/Lib/robotparser.py +++ b/Lib/robotparser.py @@ -160,6 +160,7 @@ def __init__(self, path, allowance): if path == '' and not allowance: # an empty value means allow all allowance = True + path = urlparse.urlunparse(urlparse.urlparse(path)) self.path = urllib.quote(path) self.allowance = allowance diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index b3d4a46056b..651301bde2a 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -228,6 +228,18 @@ def RobotTest(index, robots_txt, good_urls, bad_urls, RobotTest(15, doc, good, bad) +# 16. Empty query (issue #17403). Normalizing the url first. +doc = """ +User-agent: * +Allow: /some/path? +Disallow: /another/path? +""" + +good = ['/some/path?'] +bad = ['/another/path?'] + +RobotTest(16, doc, good, bad) + class NetworkTestCase(unittest.TestCase): diff --git a/Misc/NEWS b/Misc/NEWS index 00448c45af4..7e901f70066 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -15,6 +15,10 @@ Core and Builtins Library ------- +- Issue #17403: urllib.parse.robotparser normalizes the urls before adding to + ruleline. This helps in handling certain types invalid urls in a conservative + manner. Patch contributed by Mher Movsisyan. + - Implement inequality on weakref.WeakSet. - Issue #17981: Closed socket on error in SysLogHandler. From 51b6a98485c5481de63b135515ddd1c82ea8de4c Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Thu, 30 May 2013 00:14:29 -0700 Subject: [PATCH 148/480] Issue #18098: The deprecated OS X Build Applet.app fails to build on OS X 10.8 systems because the Apple-deprecated QuickDraw headers have been removed from Xcode 4. Skip building it in this case. --- Lib/plat-mac/EasyDialogs.py | 9 ++++++++- Mac/Makefile.in | 25 ++++++++++++++++--------- Mac/README | 8 +++++--- Misc/NEWS | 7 +++++++ configure | 2 +- configure.ac | 2 +- 6 files changed, 38 insertions(+), 15 deletions(-) diff --git a/Lib/plat-mac/EasyDialogs.py b/Lib/plat-mac/EasyDialogs.py index 129cf2cae26..1d3edb3aacd 100644 --- a/Lib/plat-mac/EasyDialogs.py +++ b/Lib/plat-mac/EasyDialogs.py @@ -243,8 +243,15 @@ def AskYesNoCancel(question, default = 0, yes=None, no=None, cancel=None, id=262 +# The deprecated Carbon QuickDraw APIs are no longer available as of +# OS X 10.8. Raise an ImportError here in that case so that callers +# of EasyDialogs, like BuildApplet, will do the right thing. + +try: + screenbounds = Qd.GetQDGlobalsScreenBits().bounds +except AttributeError: + raise ImportError("QuickDraw APIs not available") -screenbounds = Qd.GetQDGlobalsScreenBits().bounds screenbounds = screenbounds[0]+4, screenbounds[1]+4, \ screenbounds[2]-4, screenbounds[3]-4 diff --git a/Mac/Makefile.in b/Mac/Makefile.in index 6e60b2af344..a3e00b22f91 100644 --- a/Mac/Makefile.in +++ b/Mac/Makefile.in @@ -202,15 +202,22 @@ install_IDLE: cd IDLE && make install install_BuildApplet: - $(RUNSHARED) @ARCH_RUN_32BIT@ $(BUILDPYTHON) $(srcdir)/scripts/BuildApplet.py \ - --destroot "$(DESTDIR)" \ - --python=$(prefix)/Resources/Python.app/Contents/MacOS/Python \ - --output "$(DESTDIR)$(PYTHONAPPSDIR)/Build Applet.app" \ - $(srcdir)/scripts/BuildApplet.py -ifneq ($(LIPO_32BIT_FLAGS),) - rm "$(DESTDIR)$(PYTHONAPPSDIR)/Build Applet.app/Contents/MacOS/Python" - lipo $(LIPO_32BIT_FLAGS) -output "$(DESTDIR)$(PYTHONAPPSDIR)/Build Applet.app/Contents/MacOS/Python" $(BUILDPYTHON) -endif + if ! $(RUNSHARED) @ARCH_RUN_32BIT@ $(BUILDPYTHON) \ + -c 'import EasyDialogs' 2>/dev/null ; then \ + echo "EasyDialogs not available in this Python - skipping Build Applet.app" ; \ + else \ + $(RUNSHARED) @ARCH_RUN_32BIT@ $(BUILDPYTHON) $(srcdir)/scripts/BuildApplet.py \ + --destroot "$(DESTDIR)" \ + --python=$(prefix)/Resources/Python.app/Contents/MacOS/Python \ + --output "$(DESTDIR)$(PYTHONAPPSDIR)/Build Applet.app" \ + $(srcdir)/scripts/BuildApplet.py && \ + if [ -n "$(LIPO_32BIT_FLAGS)" ] ; then \ + rm "$(DESTDIR)$(PYTHONAPPSDIR)/Build Applet.app/Contents/MacOS/Python" && \ + lipo $(LIPO_32BIT_FLAGS) \ + -output "$(DESTDIR)$(PYTHONAPPSDIR)/Build Applet.app/Contents/MacOS/Python" \ + $(BUILDPYTHON) ; \ + fi \ + fi MACLIBDEST=$(LIBDEST)/plat-mac MACTOOLSDEST=$(prefix)/Mac/Tools diff --git a/Mac/README b/Mac/README index 555b7ca7030..0a07b5ebc95 100644 --- a/Mac/README +++ b/Mac/README @@ -209,9 +209,11 @@ script to set runtime options. These options can be set once and for all through PythonLauncher's preferences dialog. "BuildApplet.app" creates an applet from a Python script. Drop the script on it -and out comes a full-featured MacOS application. There is much more to this, -to be supplied later. Some useful (but outdated) info can be found in -Mac/Demo. +and out comes a full-featured MacOS application. BuildApplet.app is now +deprecated and has been removed in Python 3. As of OS X 10.8, Xcode 4 no +longer supplies the headers for the deprecated QuickDraw APIs used by +the EasyDialogs module making BuildApplet unusable as an app. It will +not be built by the Mac/Makefile in this case. The commandline scripts /usr/local/bin/python and pythonw can be used to run non-GUI and GUI python scripts from the command line, respectively. diff --git a/Misc/NEWS b/Misc/NEWS index 7e901f70066..8804533d7e0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -37,6 +37,13 @@ Library the default for linking if LDSHARED is not also overriden. This restores Distutils behavior introduced in 2.7.3 and inadvertently dropped in 2.7.4. +Build +----- + +- Issue #18098: The deprecated OS X Build Applet.app fails to build on + OS X 10.8 systems because the Apple-deprecated QuickDraw headers have + been removed from Xcode 4. Skip building it in this case. + IDLE ---- diff --git a/configure b/configure index dc0dfd04ad1..644e76c13fb 100755 --- a/configure +++ b/configure @@ -2981,6 +2981,7 @@ fi +ARCH_RUN_32BIT="" UNIVERSAL_ARCHS="32-bit" @@ -7996,7 +7997,6 @@ case $ac_sys_system/$ac_sys_release in esac -ARCH_RUN_32BIT="" case $ac_sys_system/$ac_sys_release in Darwin/[01567]\..*) diff --git a/configure.ac b/configure.ac index 30f5bf404a3..4196bc00b25 100644 --- a/configure.ac +++ b/configure.ac @@ -145,6 +145,7 @@ fi AC_SUBST(UNIVERSALSDK) AC_SUBST(ARCH_RUN_32BIT) +ARCH_RUN_32BIT="" UNIVERSAL_ARCHS="32-bit" AC_SUBST(LIPO_32BIT_FLAGS) @@ -1801,7 +1802,6 @@ case $ac_sys_system/$ac_sys_release in esac -ARCH_RUN_32BIT="" AC_SUBST(LIBTOOL_CRUFT) case $ac_sys_system/$ac_sys_release in Darwin/@<:@01567@:>@\..*) From 5783febe1bf6fb7815d33fa6e34ac763498b2e44 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Thu, 30 May 2013 14:47:33 -0400 Subject: [PATCH 149/480] Issue #15392: Create a unittest framework for IDLE, 2.7 version. Preliminary patch by Rajagopalasarma Jayakrishnan. --- Lib/idlelib/CallTips.py | 4 +- Lib/idlelib/PathBrowser.py | 3 +- Lib/idlelib/idle_test/README.txt | 63 +++++++++++++++++++++++ Lib/idlelib/idle_test/__init__.py | 9 ++++ Lib/idlelib/idle_test/test_calltips.py | 14 +++++ Lib/idlelib/idle_test/test_pathbrowser.py | 12 +++++ Lib/test/test_idle.py | 17 ++++++ Misc/ACKS | 1 + Misc/NEWS | 3 ++ 9 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 Lib/idlelib/idle_test/README.txt create mode 100644 Lib/idlelib/idle_test/__init__.py create mode 100644 Lib/idlelib/idle_test/test_calltips.py create mode 100644 Lib/idlelib/idle_test/test_pathbrowser.py create mode 100644 Lib/test/test_idle.py diff --git a/Lib/idlelib/CallTips.py b/Lib/idlelib/CallTips.py index d533ce13b95..0fdef11c699 100644 --- a/Lib/idlelib/CallTips.py +++ b/Lib/idlelib/CallTips.py @@ -223,4 +223,6 @@ def test(tests): tests = (t1, t2, t3, t4, t5, t6, t7, TC, tc.t1, tc.t2, tc.t3, tc.t4, tc.t5, tc.t6, tc.t7) - test(tests) + # test(tests) + from unittest import main + main('idlelib.idle_test.test_calltips', verbosity=2, exit=False) diff --git a/Lib/idlelib/PathBrowser.py b/Lib/idlelib/PathBrowser.py index d88a48e344e..0f4fb3e8ba0 100644 --- a/Lib/idlelib/PathBrowser.py +++ b/Lib/idlelib/PathBrowser.py @@ -92,4 +92,5 @@ def main(): mainloop() if __name__ == "__main__": - main() + from unittest import main + main('idlelib.idle_test.test_pathbrowser', verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/README.txt b/Lib/idlelib/idle_test/README.txt new file mode 100644 index 00000000000..1919f2f52b1 --- /dev/null +++ b/Lib/idlelib/idle_test/README.txt @@ -0,0 +1,63 @@ +README FOR IDLE TESTS IN IDLELIB.IDLE_TEST + +The idle directory, idlelib, has over 60 xyz.py files. The idle_test +subdirectory should contain a test_xyy.py for each one. (For test modules, +make 'xyz' lower case.) Each should start with the following cut-paste +template, with the blanks after after '.'. 'as', and '_' filled in. +--- +import unittest +import idlelib. as + +class Test_(unittest.TestCase): + + def test_(self): + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) +--- +Idle tests are run with unittest; do not use regrtest's test_main. + +Once test_xyy is written, the following should go at the end of xyy.py, +with xyz (lowercased) added after 'test_'. +--- +if __name__ == "__main__": + import unittest + unittest.main('idlelib.idle_test.test_', verbosity=2, exit=False) +--- + +In Idle, pressing F5 in an editor window with either xyz.py or test_xyz.py +loaded will then run the test with the version of Python running Idle and +tracebacks will appear in the Shell window. The options are appropriate for +developers running (as opposed to importing) either type of file during +development: verbosity=2 lists all test_y methods; exit=False avoids a +spurious sys.exit traceback when running in Idle. The following command +lines also run test_xyz.py + +python -m idlelib.xyz # With the capitalization of the xyz module +python -m unittest -v idlelib.idle_test.test_xyz + +To run all idle tests either interactively ('>>>', with unittest imported) +or from a command line, use one of the following. + +>>> unittest.main('idlelib.idle_test', verbosity=2, exit=False) +python -m unittest -v idlelib.idle_test +python -m test.test_idle +python -m test test_idle + +The idle tests are 'discovered' in idlelib.idle_test.__init__.load_tests, +which is also imported into test.test_idle. Normally, neither file should be +changed when working on individual test modules. The last command runs runs +unittest indirectly through regrtest. The same happens when the entire test +suite is run with 'python -m test'. So it must work for buildbots to stay green. + +To run an individual Testcase or test method, extend the +dotted name given to unittest on the command line. + +python -m unittest -v idlelib.idle_test.text_xyz.Test_case.test_meth + +To disable test/test_idle.py, there are at least two choices. +a. Comment out 'load_tests' line, no no tests are discovered (simple and safe); +Running no tests passes, so there is no indication that nothing was run. +b.Before that line, make module an unexpected skip for regrtest with +import unittest; raise unittest.SkipTest('skip for buildbots') +When run directly with unittest, this causes a normal exit and traceback. \ No newline at end of file diff --git a/Lib/idlelib/idle_test/__init__.py b/Lib/idlelib/idle_test/__init__.py new file mode 100644 index 00000000000..1bc953643e1 --- /dev/null +++ b/Lib/idlelib/idle_test/__init__.py @@ -0,0 +1,9 @@ +from os.path import dirname + +def load_tests(loader, standard_tests, pattern): + this_dir = dirname(__file__) + top_dir = dirname(dirname(this_dir)) + package_tests = loader.discover(start_dir=this_dir, pattern='test*.py', + top_level_dir=top_dir) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py new file mode 100644 index 00000000000..1733d392b35 --- /dev/null +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -0,0 +1,14 @@ +import unittest +import idlelib.CallTips as ct +CTi = ct.CallTips() + +class Test_get_entity(unittest.TestCase): + # In 3.x, get_entity changed from 'instance method' to module function + # since 'self' not used. Use dummy instance until change 2.7 also. + def test_bad_entity(self): + self.assertIsNone(CTi.get_entity('1/0')) + def test_good_entity(self): + self.assertIs(CTi.get_entity('int'), int) + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_pathbrowser.py b/Lib/idlelib/idle_test/test_pathbrowser.py new file mode 100644 index 00000000000..7ad7c97a79e --- /dev/null +++ b/Lib/idlelib/idle_test/test_pathbrowser.py @@ -0,0 +1,12 @@ +import unittest +import idlelib.PathBrowser as PathBrowser + +class PathBrowserTest(unittest.TestCase): + + def test_DirBrowserTreeItem(self): + # Issue16226 - make sure that getting a sublist works + d = PathBrowser.DirBrowserTreeItem('') + d.GetSubList() + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py new file mode 100644 index 00000000000..c48dccb410c --- /dev/null +++ b/Lib/test/test_idle.py @@ -0,0 +1,17 @@ +# Skip test if _tkinter or _thread wasn't built or idlelib was deleted. +try: + import Tkinter + import threading # imported by PyShell, imports _thread + import idlelib.idle_test as idletest +except ImportError: + import unittest + raise unittest.SkipTest + +# Without test_main present, regrtest.runtest_inner (line1219) calls +# unittest.TestLoader().loadTestsFromModule(this_module) which calls +# load_tests() if it finds it. (Unittest.main does the same.) +load_tests = idletest.load_tests + +if __name__ == '__main__': + import unittest + unittest.main(verbosity=2, exit=False) diff --git a/Misc/ACKS b/Misc/ACKS index 21a1a49381a..484202699e1 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -478,6 +478,7 @@ Geert Jansen Jack Jansen Bill Janssen Thomas Jarosch +Rajagopalasarma Jayakrishnan Drew Jenkins Flemming Kjær Jensen Philip H. Jensen diff --git a/Misc/NEWS b/Misc/NEWS index 8804533d7e0..59937455391 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -47,6 +47,9 @@ Build IDLE ---- +- Issue #15392: Create a unittest framework for IDLE. + Preliminary patch by Rajagopalasarma Jayakrishnan + - Issue #14146: Highlight source line while debugging on Windows. - Issue #17532: Always include Options menu for IDLE on OS X. From 4965dafd7b3831ca39ba42845992d890cf903355 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Thu, 30 May 2013 18:24:28 -0400 Subject: [PATCH 150/480] Issue #15392: Use test.test_support, as used test.support in 3.x. --- Lib/test/test_idle.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index c48dccb410c..9eeb4e2e180 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -1,11 +1,8 @@ # Skip test if _tkinter or _thread wasn't built or idlelib was deleted. -try: - import Tkinter - import threading # imported by PyShell, imports _thread - import idlelib.idle_test as idletest -except ImportError: - import unittest - raise unittest.SkipTest +from test.test_support import import_module +import_module('Tkinter') +import_module('threading') # imported by PyShell, imports _thread +idletest = import_module('idlelib.idle_test') # Without test_main present, regrtest.runtest_inner (line1219) calls # unittest.TestLoader().loadTestsFromModule(this_module) which calls From c924b297d5f3d1c7791d10688e239e9c2a4ccc5f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 31 May 2013 22:34:53 +0300 Subject: [PATCH 151/480] Issue #18094: test_uuid no more reports skipped tests as passed. --- Lib/test/test_uuid.py | 69 ++++++++++++++++--------------------------- Misc/NEWS | 2 ++ 2 files changed, 28 insertions(+), 43 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index e5030849184..6fe2fe5edb3 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1,5 +1,6 @@ -from unittest import TestCase +import unittest from test import test_support +import os import uuid def importable(name): @@ -9,7 +10,7 @@ def importable(name): except: return False -class TestUUID(TestCase): +class TestUUID(unittest.TestCase): last_node = None source2node = {} @@ -299,24 +300,22 @@ def check_node(self, node, source): else: TestUUID.last_node = node + @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_ifconfig_getnode(self): - import sys - import os - if os.name == 'posix': - node = uuid._ifconfig_getnode() - if node is not None: - self.check_node(node, 'ifconfig') + node = uuid._ifconfig_getnode() + if node is not None: + self.check_node(node, 'ifconfig') + @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_ipconfig_getnode(self): - import os - if os.name == 'nt': - node = uuid._ipconfig_getnode() - if node is not None: - self.check_node(node, 'ipconfig') + node = uuid._ipconfig_getnode() + if node is not None: + self.check_node(node, 'ipconfig') + @unittest.skipUnless(importable('win32wnet'), 'requires win32wnet') + @unittest.skipUnless(importable('netbios'), 'requires netbios') def test_netbios_getnode(self): - if importable('win32wnet') and importable('netbios'): - self.check_node(uuid._netbios_getnode(), 'netbios') + self.check_node(uuid._netbios_getnode(), 'netbios') def test_random_getnode(self): node = uuid._random_getnode() @@ -324,22 +323,20 @@ def test_random_getnode(self): self.assertTrue(node & 0x010000000000) self.assertTrue(node < (1L << 48)) + @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipUnless(importable('ctypes'), 'requires ctypes') def test_unixdll_getnode(self): - import sys - import os - if importable('ctypes') and os.name == 'posix': - try: # Issues 1481, 3581: _uuid_generate_time() might be None. - self.check_node(uuid._unixdll_getnode(), 'unixdll') - except TypeError: - pass + try: # Issues 1481, 3581: _uuid_generate_time() might be None. + self.check_node(uuid._unixdll_getnode(), 'unixdll') + except TypeError: + pass + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + @unittest.skipUnless(importable('ctypes'), 'requires ctypes') def test_windll_getnode(self): - import os - if importable('ctypes') and os.name == 'nt': - self.check_node(uuid._windll_getnode(), 'windll') + self.check_node(uuid._windll_getnode(), 'windll') def test_getnode(self): - import sys node1 = uuid.getnode() self.check_node(node1, "getnode1") @@ -349,13 +346,8 @@ def test_getnode(self): self.assertEqual(node1, node2) + @unittest.skipUnless(importable('ctypes'), 'requires ctypes') def test_uuid1(self): - # uuid1 requires ctypes. - try: - import ctypes - except ImportError: - return - equal = self.assertEqual # Make sure uuid1() generates UUIDs that are actually version 1. @@ -408,13 +400,8 @@ def test_uuid3(self): equal(u, uuid.UUID(v)) equal(str(u), v) + @unittest.skipUnless(importable('ctypes'), 'requires ctypes') def test_uuid4(self): - # uuid4 requires ctypes. - try: - import ctypes - except ImportError: - return - equal = self.assertEqual # Make sure uuid4() generates UUIDs that are actually version 4. @@ -446,12 +433,8 @@ def test_uuid5(self): equal(u, uuid.UUID(v)) equal(str(u), v) + @unittest.skipUnless(os.name == 'posix', 'requires Posix') def testIssue8621(self): - import os - import sys - if os.name != 'posix': - return - # On at least some versions of OSX uuid.uuid4 generates # the same sequence of UUIDs in the parent and any # children started using fork. diff --git a/Misc/NEWS b/Misc/NEWS index 59937455391..93d2c0956ee 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -58,6 +58,8 @@ IDLE Tests ----- +- Issue #18094: test_uuid no more reports skipped tests as passed. + - Issue #11995: test_pydoc doesn't import all sys.path modules anymore. Documentation From 60369a83d1279966a568a704c6e4228ae33b76e4 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 1 Jun 2013 07:59:10 -0700 Subject: [PATCH 152/480] Fix #16450 test_missing_localfile testcase fails on misconfigured hostname. Refactor test to accomodate that and exercise the needed functionality. --- Lib/test/test_urllib.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 3a273f873b1..e60adb23357 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -227,13 +227,13 @@ def test_missing_localfile(self): 'file://localhost/a/missing/file.py') fd, tmp_file = tempfile.mkstemp() tmp_fileurl = 'file://localhost/' + tmp_file.replace(os.path.sep, '/') + self.assertTrue(os.path.exists(tmp_file)) try: - self.assertTrue(os.path.exists(tmp_file)) fp = urllib.urlopen(tmp_fileurl) + fp.close() finally: os.close(fd) - fp.close() - os.unlink(tmp_file) + os.unlink(tmp_file) self.assertFalse(os.path.exists(tmp_file)) self.assertRaises(IOError, urllib.urlopen, tmp_fileurl) From 76176893ab3159b740e417c8f186740bd7450f5e Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 1 Jun 2013 08:24:31 -0700 Subject: [PATCH 153/480] Fix #17967: For ftp urls CWD to target instead of hopping to each directory towards target. This fixes a bug where target is accessible, but parent directories are restricted. --- Lib/urllib.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/urllib.py b/Lib/urllib.py index f9655f9e88a..32b4919f07f 100644 --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -870,8 +870,7 @@ def init(self): self.ftp = ftplib.FTP() self.ftp.connect(self.host, self.port, self.timeout) self.ftp.login(self.user, self.passwd) - for dir in self.dirs: - self.ftp.cwd(dir) + self.ftp.cwd(os.path.join(*self.dirs)) def retrfile(self, file, type): import ftplib From dd24e812533ff858b6f63c01678ed014d768e17a Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 1 Jun 2013 11:11:30 -0700 Subject: [PATCH 154/480] Fix thishost helper funtion in urllib. Returns the ipaddress of localhost when hostname is resolvable by socket.gethostname for local machine. This all fixes certain freebsd builtbot failures. --- Lib/urllib.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/urllib.py b/Lib/urllib.py index 32b4919f07f..09a054b623c 100644 --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -819,7 +819,10 @@ def thishost(): """Return the IP address of the current host.""" global _thishost if _thishost is None: - _thishost = socket.gethostbyname(socket.gethostname()) + try: + _thishost = socket.gethostbyname(socket.gethostname()) + except socket.gaierror: + _thishost = socket.gethostbyname('localhost') return _thishost _ftperrors = None From a45cc9af5bb244d6653bd290d1c5270e6736208d Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 2 Jun 2013 11:59:09 -0700 Subject: [PATCH 155/480] Fix #17967 - Fix related to regression on Windows. os.path.join(*self.dirs) produces an invalid path on windows. ftp paths are always forward-slash seperated like this. /pub/dir. --- Lib/urllib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/urllib.py b/Lib/urllib.py index 09a054b623c..244cb05374d 100644 --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -873,7 +873,8 @@ def init(self): self.ftp = ftplib.FTP() self.ftp.connect(self.host, self.port, self.timeout) self.ftp.login(self.user, self.passwd) - self.ftp.cwd(os.path.join(*self.dirs)) + _target = '/'.join(self.dirs) + self.ftp.cwd(_target) def retrfile(self, file, type): import ftplib From a9c12da1d354edad7f278d609d14b8204b38540b Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Wed, 5 Jun 2013 14:24:42 -0400 Subject: [PATCH 156/480] Issue18130: Test class idlelib.configSectionNameDialog.GetCfgSectionNameDialog. Fix bug in existing human test and add instructions; fix two bugs in tested code; remove redundancies, add spaces, and change two internal method names. Add mock_tk with mocks for tkinter.Variable subclasses and tkinter.messagebox. Use mocks in test_config_name to unittest methods that are otherwise gui-free. --- Lib/idlelib/configSectionNameDialog.py | 115 +++++++++++----------- Lib/idlelib/idle_test/mock_tk.py | 63 ++++++++++++ Lib/idlelib/idle_test/test_config_name.py | 75 ++++++++++++++ 3 files changed, 197 insertions(+), 56 deletions(-) create mode 100644 Lib/idlelib/idle_test/mock_tk.py create mode 100644 Lib/idlelib/idle_test/test_config_name.py diff --git a/Lib/idlelib/configSectionNameDialog.py b/Lib/idlelib/configSectionNameDialog.py index 4f1b002afca..04fcd8cf9a9 100644 --- a/Lib/idlelib/configSectionNameDialog.py +++ b/Lib/idlelib/configSectionNameDialog.py @@ -1,97 +1,100 @@ """ Dialog that allows user to specify a new config file section name. Used to get new highlight theme and keybinding set names. +The 'return value' for the dialog, used two placed in configDialog.py, +is the .result attribute set in the Ok and Cancel methods. """ from Tkinter import * import tkMessageBox - class GetCfgSectionNameDialog(Toplevel): - def __init__(self,parent,title,message,usedNames): + def __init__(self, parent, title, message, used_names): """ message - string, informational message to display - usedNames - list, list of names already in use for validity check + used_names - string collection, names already in use for validity check """ Toplevel.__init__(self, parent) self.configure(borderwidth=5) - self.resizable(height=FALSE,width=FALSE) + self.resizable(height=FALSE, width=FALSE) self.title(title) self.transient(parent) self.grab_set() self.protocol("WM_DELETE_WINDOW", self.Cancel) self.parent = parent - self.message=message - self.usedNames=usedNames - self.result='' - self.CreateWidgets() - self.withdraw() #hide while setting geometry + self.message = message + self.used_names = used_names + self.create_widgets() + self.withdraw() #hide while setting geometry self.update_idletasks() #needs to be done here so that the winfo_reqwidth is valid self.messageInfo.config(width=self.frameMain.winfo_reqwidth()) - self.geometry("+%d+%d" % - ((parent.winfo_rootx()+((parent.winfo_width()/2) - -(self.winfo_reqwidth()/2)), - parent.winfo_rooty()+((parent.winfo_height()/2) - -(self.winfo_reqheight()/2)) )) ) #centre dialog over parent - self.deiconify() #geometry set, unhide + self.geometry( + "+%d+%d" % ( + parent.winfo_rootx() + + (parent.winfo_width()/2 - self.winfo_reqwidth()/2), + parent.winfo_rooty() + + (parent.winfo_height()/2 - self.winfo_reqheight()/2) + ) ) #centre dialog over parent + self.deiconify() #geometry set, unhide self.wait_window() - - def CreateWidgets(self): - self.name=StringVar(self) - self.fontSize=StringVar(self) - self.frameMain = Frame(self,borderwidth=2,relief=SUNKEN) - self.frameMain.pack(side=TOP,expand=TRUE,fill=BOTH) - self.messageInfo=Message(self.frameMain,anchor=W,justify=LEFT,padx=5,pady=5, - text=self.message)#,aspect=200) - entryName=Entry(self.frameMain,textvariable=self.name,width=30) + def create_widgets(self): + self.name = StringVar(self.parent) + self.fontSize = StringVar(self.parent) + self.frameMain = Frame(self, borderwidth=2, relief=SUNKEN) + self.frameMain.pack(side=TOP, expand=TRUE, fill=BOTH) + self.messageInfo = Message(self.frameMain, anchor=W, justify=LEFT, + padx=5, pady=5, text=self.message) #,aspect=200) + entryName = Entry(self.frameMain, textvariable=self.name, width=30) entryName.focus_set() - self.messageInfo.pack(padx=5,pady=5)#,expand=TRUE,fill=BOTH) - entryName.pack(padx=5,pady=5) - frameButtons=Frame(self) - frameButtons.pack(side=BOTTOM,fill=X) - self.buttonOk = Button(frameButtons,text='Ok', - width=8,command=self.Ok) - self.buttonOk.grid(row=0,column=0,padx=5,pady=5) - self.buttonCancel = Button(frameButtons,text='Cancel', - width=8,command=self.Cancel) - self.buttonCancel.grid(row=0,column=1,padx=5,pady=5) + self.messageInfo.pack(padx=5, pady=5) #, expand=TRUE, fill=BOTH) + entryName.pack(padx=5, pady=5) + frameButtons = Frame(self, pady=2) + frameButtons.pack(side=BOTTOM) + self.buttonOk = Button(frameButtons, text='Ok', + width=8, command=self.Ok) + self.buttonOk.pack(side=LEFT, padx=5) + self.buttonCancel = Button(frameButtons, text='Cancel', + width=8, command=self.Cancel) + self.buttonCancel.pack(side=RIGHT, padx=5) - def NameOk(self): - #simple validity check for a sensible - #ConfigParser file section name - nameOk=1 - name=self.name.get() - name.strip() + def name_ok(self): + ''' After stripping entered name, check that it is a sensible + ConfigParser file section name. Return it if it is, '' if not. + ''' + name = self.name.get().strip() if not name: #no name specified tkMessageBox.showerror(title='Name Error', message='No name specified.', parent=self) - nameOk=0 elif len(name)>30: #name too long tkMessageBox.showerror(title='Name Error', message='Name too long. It should be no more than '+ '30 characters.', parent=self) - nameOk=0 - elif name in self.usedNames: + name = '' + elif name in self.used_names: tkMessageBox.showerror(title='Name Error', message='This name is already in use.', parent=self) - nameOk=0 - return nameOk - + name = '' + return name def Ok(self, event=None): - if self.NameOk(): - self.result=self.name.get().strip() + name = self.name_ok() + if name: + self.result = name self.destroy() - def Cancel(self, event=None): - self.result='' + self.result = '' self.destroy() - if __name__ == '__main__': - #test the dialog - root=Tk() + import unittest + unittest.main('idlelib.idle_test.test_config_name', verbosity=2, exit=False) + + # also human test the dialog + root = Tk() def run(): - keySeq='' dlg=GetCfgSectionNameDialog(root,'Get Name', - 'The information here should need to be word wrapped. Test.') + "After the text entered with [Ok] is stripped, , " + "'abc', or more that 30 chars are errors. " + "Close with a valid entry (printed), [Cancel], or [X]", + {'abc'}) print dlg.result - Button(root,text='Dialog',command=run).pack() + Message(root, text='').pack() # will be needed for oher dialog tests + Button(root, text='Click to begin dialog test', command=run).pack() root.mainloop() diff --git a/Lib/idlelib/idle_test/mock_tk.py b/Lib/idlelib/idle_test/mock_tk.py new file mode 100644 index 00000000000..60724a18c39 --- /dev/null +++ b/Lib/idlelib/idle_test/mock_tk.py @@ -0,0 +1,63 @@ +"""Classes that replace tkinter gui objects used by an object being tested. +A gui object is anything with a master or parent paramenter, which is typically +required in spite of what the doc strings say. +""" + +class Var(object): + "Use for String/Int/BooleanVar: incomplete" + def __init__(self, master=None, value=None, name=None): + self.master = master + self.value = value + self.name = name + def set(self, value): + self.value = value + def get(self): + return self.value + +class Mbox_func(object): + """Generic mock for messagebox functions. All have same call signature. + Mbox instantiates once for each function. Tester uses attributes. + """ + def __init__(self): + self.result = None # The return for all show funcs + def __call__(self, title, message, *args, **kwds): + # Save all args for possible examination by tester + self.title = title + self.message = message + self.args = args + self.kwds = kwds + return self.result # Set by tester for ask functions + +class Mbox(object): + """Mock for tkinter.messagebox with an Mbox_func for each function. + This module was 'tkMessageBox' in 2.x; hence the 'import as' in 3.x. + Example usage in test_module.py for testing functios in module.py: + --- +from idlelib.idle_test.mock_tk import Mbox +import module + +orig_mbox = module.tkMessageBox +showerror = Mbox.showerror # example, for attribute access in test methods + +class Test(unittest.TestCase): + + @classmethod + def setUpClass(cls): + module.tkMessageBox = Mbox + + @classmethod + def tearDownClass(cls): + module.tkMessageBox = orig_mbox + --- + When tkMessageBox functions are the only gui making calls in a method, + this replacement makes the method gui-free and unit-testable. + For 'ask' functions, set func.result return before calling method. + """ + askokcancel = Mbox_func() # True or False + askquestion = Mbox_func() # 'yes' or 'no' + askretrycancel = Mbox_func() # True or False + askyesno = Mbox_func() # True or False + askyesnocancel = Mbox_func() # True, False, or None + showerror = Mbox_func() # None + showinfo = Mbox_func() # None + showwarning = Mbox_func() # None diff --git a/Lib/idlelib/idle_test/test_config_name.py b/Lib/idlelib/idle_test/test_config_name.py new file mode 100644 index 00000000000..a38c896e58e --- /dev/null +++ b/Lib/idlelib/idle_test/test_config_name.py @@ -0,0 +1,75 @@ +"""Unit tests for idlelib.configSectionNameDialog""" +import unittest +from idlelib.idle_test.mock_tk import Var, Mbox +from idlelib import configSectionNameDialog as name_dialog_module + +name_dialog = name_dialog_module.GetCfgSectionNameDialog + +class Dummy_name_dialog(object): + # Mock for testing the following methods of name_dialog + name_ok = name_dialog.name_ok.im_func + Ok = name_dialog.Ok.im_func + Cancel = name_dialog.Cancel.im_func + # Attributes, constant or variable, needed for tests + used_names = ['used'] + name = Var() + result = None + destroyed = False + def destroy(self): + self.destroyed = True + +# name_ok calls Mbox.showerror if name is not ok +orig_mbox = name_dialog_module.tkMessageBox +showerror = Mbox.showerror + +class TestConfigName(unittest.TestCase): + dialog = Dummy_name_dialog() + + @classmethod + def setUpClass(cls): + name_dialog_module.tkMessageBox = Mbox + + @classmethod + def tearDownClass(cls): + name_dialog_module.tkMessageBox = orig_mbox + + def test_blank_name(self): + self.dialog.name.set(' ') + self.assertEqual(self.dialog.name_ok(), '') + self.assertEqual(showerror.title, 'Name Error') + self.assertIn('No', showerror.message) + + def test_used_name(self): + self.dialog.name.set('used') + self.assertEqual(self.dialog.name_ok(), '') + self.assertEqual(showerror.title, 'Name Error') + self.assertIn('use', showerror.message) + + def test_long_name(self): + self.dialog.name.set('good'*8) + self.assertEqual(self.dialog.name_ok(), '') + self.assertEqual(showerror.title, 'Name Error') + self.assertIn('too long', showerror.message) + + def test_good_name(self): + self.dialog.name.set(' good ') + showerror.title = 'No Error' # should not be called + self.assertEqual(self.dialog.name_ok(), 'good') + self.assertEqual(showerror.title, 'No Error') + + def test_ok(self): + self.dialog.destroyed = False + self.dialog.name.set('good') + self.dialog.Ok() + self.assertEqual(self.dialog.result, 'good') + self.assertTrue(self.dialog.destroyed) + + def test_cancel(self): + self.dialog.destroyed = False + self.dialog.Cancel() + self.assertEqual(self.dialog.result, '') + self.assertTrue(self.dialog.destroyed) + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) From 28fb97bc9b62eac3ecfd563ea39d96789dca0660 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 9 Jun 2013 01:04:21 +0300 Subject: [PATCH 157/480] #18126: update NumPy links in the documentation. Patch by Yury V. Zaytsev. --- Doc/faq/programming.rst | 2 +- Doc/library/array.rst | 6 ++---- Misc/ACKS | 1 + 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index b9ed53410c7..16fabe1ddf6 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1204,7 +1204,7 @@ use a list comprehension:: A = [[None] * w for i in range(h)] Or, you can use an extension that provides a matrix datatype; `Numeric Python -`_ is the best known. +`_ is the best known. How do I apply a method to a sequence of objects? diff --git a/Doc/library/array.rst b/Doc/library/array.rst index d34cf382b1e..1766d47c0fa 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -268,9 +268,7 @@ Examples:: Packing and unpacking of External Data Representation (XDR) data as used in some remote procedure call systems. - `The Numerical Python Manual `_ + `The Numerical Python Documentation `_ The Numeric Python extension (NumPy) defines another array type; see - http://numpy.sourceforge.net/ for further information about Numerical Python. - (A PDF version of the NumPy manual is available at - http://numpy.sourceforge.net/numdoc/numdoc.pdf). + http://www.numpy.org/ for further information about Numerical Python. diff --git a/Misc/ACKS b/Misc/ACKS index 484202699e1..92fffc68de3 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1139,6 +1139,7 @@ Moshe Zadka Milan Zamazal Artur Zaprzala Mike Zarnstorff +Yury V. Zaytsev Siebren van der Zee Nickolai Zeldovich Uwe Zessin From 2e215c346a190a9ebc1544f5816190772115e89b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 9 Jun 2013 16:54:56 +0300 Subject: [PATCH 158/480] Issue #18038: SyntaxError raised during compilation sources with illegal encoding now always contains an encoding name. --- Lib/test/test_pep263.py | 18 ++++++++++++++++++ Misc/NEWS | 3 +++ Parser/tokenizer.c | 14 +++++++------- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_pep263.py b/Lib/test/test_pep263.py index 9286467adf3..4b60624be21 100644 --- a/Lib/test/test_pep263.py +++ b/Lib/test/test_pep263.py @@ -41,6 +41,24 @@ def test_issue7820(self): # two bytes in common with the UTF-8 BOM self.assertRaises(SyntaxError, eval, '\xef\xbb\x20') + def test_error_message(self): + compile('# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec') + compile('\xef\xbb\xbf\n', 'dummy', 'exec') + compile('\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec') + with self.assertRaisesRegexp(SyntaxError, 'fake'): + compile('# -*- coding: fake -*-\n', 'dummy', 'exec') + with self.assertRaisesRegexp(SyntaxError, 'iso-8859-15'): + compile('\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', + 'dummy', 'exec') + with self.assertRaisesRegexp(SyntaxError, 'BOM'): + compile('\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', + 'dummy', 'exec') + with self.assertRaisesRegexp(SyntaxError, 'fake'): + compile('\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec') + with self.assertRaisesRegexp(SyntaxError, 'BOM'): + compile('\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec') + + def test_main(): test_support.run_unittest(PEP263Test) diff --git a/Misc/NEWS b/Misc/NEWS index 93d2c0956ee..784011a2015 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -9,6 +9,9 @@ What's New in Python 2.7.6? Core and Builtins ----------------- +- Issue #18038: SyntaxError raised during compilation sources with illegal + encoding now always contains an encoding name. + - Issue #18019: Fix crash in the repr of dictionaries containing their own views. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index ee6313b311d..46cf9b297cd 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -277,8 +277,11 @@ check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, tok->encoding = cs; tok->decoding_state = -1; } - else + else { + PyErr_Format(PyExc_SyntaxError, + "encoding problem: %s", cs); PyMem_FREE(cs); + } #else /* Without Unicode support, we cannot process the coding spec. Since there @@ -289,15 +292,12 @@ check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, } } else { /* then, compare cs with BOM */ r = (strcmp(tok->encoding, cs) == 0); + if (!r) + PyErr_Format(PyExc_SyntaxError, + "encoding problem: %s with BOM", cs); PyMem_FREE(cs); } } - if (!r) { - cs = tok->encoding; - if (!cs) - cs = "with BOM"; - PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs); - } return r; } From a7ee481c235cdf07b475d490958cfde4fb2b13f2 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 10 Jun 2013 10:35:36 +0200 Subject: [PATCH 159/480] Ensure that the fix for #17269 also works on OSX 10.4 AI_NUMERICSERV isn't defined on OSX 10.4. --- Lib/test/test_socket.py | 3 ++- Modules/socketmodule.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 3b112e68384..6e7218034d9 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -666,7 +666,8 @@ def testGetaddrinfo(self): socket.AI_PASSIVE) # Issue 17269 - socket.getaddrinfo("localhost", None, 0, 0, 0, socket.AI_NUMERICSERV) + if hasattr(socket, 'AI_NUMERICSERV'): + socket.getaddrinfo("localhost", None, 0, 0, 0, socket.AI_NUMERICSERV) def check_sendall_interrupted(self, with_timeout): # socketpair() is not stricly required, but it makes things easier. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 238c849b845..2735ecc7735 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -4179,7 +4179,7 @@ socket_getaddrinfo(PyObject *self, PyObject *args) "getaddrinfo() argument 2 must be integer or string"); goto err; } -#ifdef __APPLE__ +#if defined(__APPLE__) && defined(AI_NUMERICSERV) if ((flags & AI_NUMERICSERV) && (pptr == NULL || (pptr[0] == '0' && pptr[1] == 0))) { /* On OSX upto at least OSX 10.8 getaddrinfo crashes * if AI_NUMERICSERV is set and the servname is NULL or "0". From 094361a79285cbeb52f4748c2642e14cd8eb5b45 Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Mon, 10 Jun 2013 16:27:45 +0100 Subject: [PATCH 160/480] Issue #18174: Fix fd leaks in tests. --- Lib/test/test_openpty.py | 2 ++ Lib/test/test_subprocess.py | 3 ++- Lib/test/test_uuid.py | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_openpty.py b/Lib/test/test_openpty.py index 20c4fe239c2..4b34b3a3c77 100644 --- a/Lib/test/test_openpty.py +++ b/Lib/test/test_openpty.py @@ -10,6 +10,8 @@ class OpenptyTest(unittest.TestCase): def test(self): master, slave = os.openpty() + self.addCleanup(os.close, master) + self.addCleanup(os.close, slave) if not os.isatty(slave): self.fail("Slave-end of pty is not a terminal.") diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index e89d84f4903..f43b51caf83 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -806,7 +806,8 @@ def _execute_child( self._testcase.assertNotIn( fd, (p2cwrite, c2pread, errread)) finally: - map(os.close, devzero_fds) + for fd in devzero_fds: + os.close(fd) @unittest.skipIf(not os.path.exists("/dev/zero"), "/dev/zero required.") def test_preexec_errpipe_does_not_double_close_pipes(self): diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 6fe2fe5edb3..9de3d789c5e 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -448,6 +448,7 @@ def testIssue8621(self): else: os.close(fds[1]) + self.addCleanup(os.close, fds[0]) parent_value = uuid.uuid4().hex os.waitpid(pid, 0) child_value = os.read(fds[0], 100) From 088dee2473655176c1f0917c3a7d1daf41ed4fd3 Mon Sep 17 00:00:00 2001 From: Roger Serwy Date: Mon, 10 Jun 2013 23:01:20 -0500 Subject: [PATCH 161/480] #17511: Keep IDLE find dialog open after clicking "Find Next". Original patch by Sarah K. --- Lib/idlelib/SearchDialog.py | 5 ++--- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/idlelib/SearchDialog.py b/Lib/idlelib/SearchDialog.py index 7c70b84ee46..3ba0cdb95d1 100644 --- a/Lib/idlelib/SearchDialog.py +++ b/Lib/idlelib/SearchDialog.py @@ -24,13 +24,12 @@ class SearchDialog(SearchDialogBase): def create_widgets(self): f = SearchDialogBase.create_widgets(self) - self.make_button("Find", self.default_command, 1) + self.make_button("Find Next", self.default_command, 1) def default_command(self, event=None): if not self.engine.getprog(): return - if self.find_again(self.text): - self.close() + self.find_again(self.text) def find_again(self, text): if not self.engine.getpat(): diff --git a/Misc/ACKS b/Misc/ACKS index 92fffc68de3..15daf505aa2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -502,6 +502,7 @@ Jens B. Jorgensen Sijin Joseph Andreas Jung Tattoo Mabonzo K. +Sarah K. Bohuslav Kabrda Bob Kahn Kurt B. Kaiser diff --git a/Misc/NEWS b/Misc/NEWS index 784011a2015..1e28c3883ee 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -50,6 +50,9 @@ Build IDLE ---- +- Issue #17511: Keep IDLE find dialog open after clicking "Find Next". + Original patch by Sarah K. + - Issue #15392: Create a unittest framework for IDLE. Preliminary patch by Rajagopalasarma Jayakrishnan From 65838cfa241713418d338ba9648c1c016fb991bb Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 11 Jun 2013 15:00:21 -0700 Subject: [PATCH 162/480] Issue #18186: remove obsolete 2.2 compatibility comment. --- Lib/subprocess.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 309f9a39728..7fd1b025c28 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -2,8 +2,6 @@ # # For more information about this module, see PEP 324. # -# This module should remain compatible with Python 2.2, see PEP 291. -# # Copyright (c) 2003-2005 by Peter Astrand # # Licensed to PSF under a Contributor Agreement. From 01b69551acbfed1f1780a8f50e5c261f58adc41c Mon Sep 17 00:00:00 2001 From: Roger Serwy Date: Tue, 11 Jun 2013 22:13:17 -0500 Subject: [PATCH 163/480] #5492: Avoid traceback when exiting IDLE caused by a race condition. --- Lib/idlelib/PyShell.py | 11 +++++------ Misc/NEWS | 2 ++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/idlelib/PyShell.py b/Lib/idlelib/PyShell.py index 0a77bb58536..1ae69499bc4 100644 --- a/Lib/idlelib/PyShell.py +++ b/Lib/idlelib/PyShell.py @@ -370,6 +370,7 @@ def __init__(self, tkconsole): self.port = PORT self.original_compiler_flags = self.compile.compiler.flags + _afterid = None rpcclt = None rpcpid = None @@ -497,6 +498,8 @@ def interrupt_subprocess(self): threading.Thread(target=self.__request_interrupt).start() def kill_subprocess(self): + if self._afterid is not None: + self.tkconsole.text.after_cancel(self._afterid) try: self.rpcclt.close() except AttributeError: # no socket @@ -569,8 +572,8 @@ def poll_subprocess(self): pass # Reschedule myself if not self.tkconsole.closing: - self.tkconsole.text.after(self.tkconsole.pollinterval, - self.poll_subprocess) + self._afterid = self.tkconsole.text.after( + self.tkconsole.pollinterval, self.poll_subprocess) debugger = None @@ -987,10 +990,6 @@ def close(self): self.stop_readline() self.canceled = True self.closing = True - # Wait for poll_subprocess() rescheduling to stop - self.text.after(2 * self.pollinterval, self.close2) - - def close2(self): return EditorWindow.close(self) def _close(self): diff --git a/Misc/NEWS b/Misc/NEWS index 1e28c3883ee..1479f849780 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -50,6 +50,8 @@ Build IDLE ---- +- Issue #5492: Avoid traceback when exiting IDLE caused by a race condition. + - Issue #17511: Keep IDLE find dialog open after clicking "Find Next". Original patch by Sarah K. From d85681744459f4d9c321867d6fbdcc9c0a6cfc58 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 14 Jun 2013 01:06:33 -0400 Subject: [PATCH 164/480] Fix comment blocks. Adjust blocksize to a power-of-two for better divmod computations. --- Lib/test/test_deque.py | 2 +- Modules/_collectionsmodule.c | 18 +++++++----------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 595a0c4a35c..98b203edb60 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -522,7 +522,7 @@ class C(object): @test_support.cpython_only def test_sizeof(self): - BLOCKLEN = 62 + BLOCKLEN = 64 basesize = test_support.calcobjsize('2P4PlP') blocksize = struct.calcsize('2P%dP' % BLOCKLEN) self.assertEqual(object.__sizeof__(deque()), basesize) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 371631ce266..8a43d9a028a 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -8,12 +8,13 @@ */ /* The block length may be set to any number over 1. Larger numbers - * reduce the number of calls to the memory allocator but take more - * memory. Ideally, BLOCKLEN should be set with an eye to the - * length of a cache line. + * reduce the number of calls to the memory allocator, give faster + * indexing and rotation, and reduce the link::data overhead ratio. + * Ideally, the block length should be a power-of-two for faster + * division/modulo computations during indexing. */ -#define BLOCKLEN 62 +#define BLOCKLEN 64 #define CENTER ((BLOCKLEN - 1) / 2) /* A `dequeobject` is composed of a doubly-linked list of `block` nodes. @@ -58,13 +59,8 @@ static block *freeblocks[MAXFREEBLOCKS]; static block * newblock(block *leftlink, block *rightlink, Py_ssize_t len) { block *b; - /* To prevent len from overflowing PY_SSIZE_T_MAX on 64-bit machines, we - * refuse to allocate new blocks if the current len is dangerously - * close. There is some extra margin to prevent spurious arithmetic - * overflows at various places. The following check ensures that - * the blocks allocated to the deque, in the worst case, can only - * have PY_SSIZE_T_MAX-2 entries in total. - */ + /* To prevent len from overflowing PY_SSIZE_T_MAX on 32-bit machines, we + * refuse to allocate new blocks if the current len is nearing overflow. */ if (len >= PY_SSIZE_T_MAX - 2*BLOCKLEN) { PyErr_SetString(PyExc_OverflowError, "cannot add more blocks to the deque"); From 1241ad860b39a0cc6e6c51e8d291d022646bfdeb Mon Sep 17 00:00:00 2001 From: Andrew Kuchling Date: Sat, 15 Jun 2013 13:53:10 -0400 Subject: [PATCH 165/480] #18113: Objects associated to a curses.panel object with set_userptr() were leaked. Reported by Atsuo Ishimoto. --- Lib/test/test_curses.py | 13 +++++++++++++ Misc/NEWS | 3 +++ Modules/_curses_panel.c | 4 ++++ 3 files changed, 20 insertions(+) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index fa0d469e475..387a1852d9d 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -250,6 +250,18 @@ def test_userptr_without_set(stdscr): except curses.panel.error: pass +def test_userptr_memory_leak(stdscr): + w = curses.newwin(10, 10) + p = curses.panel.new_panel(w) + obj = object() + nrefs = sys.getrefcount(obj) + for i in range(100): + p.set_userptr(obj) + + p.set_userptr(None) + if sys.getrefcount(obj) != nrefs: + raise RuntimeError, "set_userptr leaked references" + def test_resize_term(stdscr): if hasattr(curses, 'resizeterm'): lines, cols = curses.LINES, curses.COLS @@ -268,6 +280,7 @@ def main(stdscr): module_funcs(stdscr) window_funcs(stdscr) test_userptr_without_set(stdscr) + test_userptr_memory_leak(stdscr) test_resize_term(stdscr) test_issue6243(stdscr) finally: diff --git a/Misc/NEWS b/Misc/NEWS index 1479f849780..efcd20a3900 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -40,6 +40,9 @@ Library the default for linking if LDSHARED is not also overriden. This restores Distutils behavior introduced in 2.7.3 and inadvertently dropped in 2.7.4. +- Issue #18113: Fixed a refcount leak in the curses.panel module's + set_userptr() method. Reported by Atsuo Ishimoto. + Build ----- diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 04a0a28a885..dd231e767fa 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -293,6 +293,10 @@ PyCursesPanel_replace_panel(PyCursesPanelObject *self, PyObject *args) static PyObject * PyCursesPanel_set_panel_userptr(PyCursesPanelObject *self, PyObject *obj) { + PyObject *oldobj; + PyCursesInitialised; + oldobj = (PyObject *) panel_userptr(self->pan); + Py_XDECREF(oldobj); Py_INCREF(obj); return PyCursesCheckERR(set_panel_userptr(self->pan, (void*)obj), "set_panel_userptr"); From bb6295c54781f6c82be6e1882aae909631fa8683 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 17 Jun 2013 16:33:48 +0300 Subject: [PATCH 166/480] Issue #18167: cgi.FieldStorage no more fails to handle multipart/form-data when \r\n appears at end of 65535 bytes without other newlines. --- Lib/cgi.py | 9 +++++++++ Lib/test/test_cgi.py | 23 +++++++++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 35 insertions(+) diff --git a/Lib/cgi.py b/Lib/cgi.py index 67079db0d81..64ba6d16caf 100755 --- a/Lib/cgi.py +++ b/Lib/cgi.py @@ -697,6 +697,9 @@ def read_lines_to_outerboundary(self): if not line: self.done = -1 break + if delim == "\r": + line = delim + line + delim = "" if line[:2] == "--" and last_line_lfend: strippedline = line.strip() if strippedline == next: @@ -713,6 +716,12 @@ def read_lines_to_outerboundary(self): delim = "\n" line = line[:-1] last_line_lfend = True + elif line[-1] == "\r": + # We may interrupt \r\n sequences if they span the 2**16 + # byte boundary + delim = "\r" + line = line[:-1] + last_line_lfend = False else: delim = "" last_line_lfend = False diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py index f6abe976cf7..7fdd482b942 100644 --- a/Lib/test/test_cgi.py +++ b/Lib/test/test_cgi.py @@ -266,6 +266,29 @@ def test_fieldstorage_multipart(self): got = getattr(fs.list[x], k) self.assertEqual(got, exp) + def test_fieldstorage_multipart_maxline(self): + # Issue #18167 + maxline = 1 << 16 + self.maxDiff = None + def check(content): + data = """ +---123 +Content-Disposition: form-data; name="upload"; filename="fake.txt" +Content-Type: text/plain + +%s +---123-- +""".replace('\n', '\r\n') % content + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', + 'REQUEST_METHOD': 'POST', + } + self.assertEqual(gen_result(data, environ), {'upload': content}) + check('x' * (maxline - 1)) + check('x' * (maxline - 1) + '\r') + check('x' * (maxline - 1) + '\r' + 'y' * (maxline - 1)) + _qs_result = { 'key1': 'value1', 'key2': ['value2x', 'value2y'], diff --git a/Misc/NEWS b/Misc/NEWS index efcd20a3900..2bbcd7059f8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -18,6 +18,9 @@ Core and Builtins Library ------- +- Issue #18167: cgi.FieldStorage no more fails to handle multipart/form-data + when \r\n appears at end of 65535 bytes without other newlines. + - Issue #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline. This helps in handling certain types invalid urls in a conservative manner. Patch contributed by Mher Movsisyan. From 0abbbbbfe724a2bf7c3ae2fe51bd0793efc54403 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 17 Jun 2013 22:03:35 +0200 Subject: [PATCH 167/480] ctypes: AIX needs an explicit #include to get alloca() --- Modules/_ctypes/callproc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 6daf455a06e..c8ce1055485 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -75,6 +75,7 @@ #include #include "ctypes.h" +#include #if defined(_DEBUG) || defined(__MINGW32__) /* Don't use structured exception handling on Windows if this is defined. From 1d650110c867c7bd550d06ced4bf080bc19aaf88 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 18 Jun 2013 13:25:24 +0200 Subject: [PATCH 168/480] Fix for r84195: add HAVE_ALLOCA_H to configure and only include alloca.h if it's available --- Modules/_ctypes/callproc.c | 3 +++ configure | 2 +- configure.ac | 2 +- pyconfig.h.in | 3 +++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index c8ce1055485..6642dc3f4c0 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -75,7 +75,10 @@ #include #include "ctypes.h" +#ifdef HAVE_ALLOCA_H +/* AIX needs alloca.h for alloca() */ #include +#endif #if defined(_DEBUG) || defined(__MINGW32__) /* Don't use structured exception handling on Windows if this is defined. diff --git a/configure b/configure index 644e76c13fb..5f376af7756 100755 --- a/configure +++ b/configure @@ -6654,7 +6654,7 @@ sys/param.h sys/poll.h sys/select.h sys/socket.h sys/statvfs.h sys/stat.h \ sys/termio.h sys/time.h \ sys/times.h sys/types.h sys/un.h sys/utsname.h sys/wait.h pty.h libutil.h \ sys/resource.h netpacket/packet.h sysexits.h bluetooth.h \ -bluetooth/bluetooth.h linux/tipc.h spawn.h util.h +bluetooth/bluetooth.h linux/tipc.h spawn.h util.h alloca.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" diff --git a/configure.ac b/configure.ac index 4196bc00b25..b4ed0107a83 100644 --- a/configure.ac +++ b/configure.ac @@ -1519,7 +1519,7 @@ sys/param.h sys/poll.h sys/select.h sys/socket.h sys/statvfs.h sys/stat.h \ sys/termio.h sys/time.h \ sys/times.h sys/types.h sys/un.h sys/utsname.h sys/wait.h pty.h libutil.h \ sys/resource.h netpacket/packet.h sysexits.h bluetooth.h \ -bluetooth/bluetooth.h linux/tipc.h spawn.h util.h) +bluetooth/bluetooth.h linux/tipc.h spawn.h util.h alloca.h) AC_HEADER_DIRENT AC_HEADER_MAJOR diff --git a/pyconfig.h.in b/pyconfig.h.in index 231b9c8a31f..65df68aed9f 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -55,6 +55,9 @@ /* Define to 1 if you have the `alarm' function. */ #undef HAVE_ALARM +/* Define to 1 if you have the header file. */ +#undef HAVE_ALLOCA_H + /* Define this if your time.h defines altzone. */ #undef HAVE_ALTZONE From c46846cd315cc95275d791837bcd0d3c84f6d7fd Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 18 Jun 2013 22:17:48 +0200 Subject: [PATCH 169/480] Issue #18256: Compilation fix for recent AIX releases. Patch by David Edelsohn. --- Misc/ACKS | 1 + Misc/NEWS | 3 +++ Python/thread_pthread.h | 3 +++ 3 files changed, 7 insertions(+) diff --git a/Misc/ACKS b/Misc/ACKS index 15daf505aa2..fc0d9d216be 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -270,6 +270,7 @@ Josip Dzolonga Maxim Dzumanenko Walter Dörwald Hans Eckardt +David Edelsohn Grant Edwards John Ehresman Eric Eisner diff --git a/Misc/NEWS b/Misc/NEWS index 2bbcd7059f8..941818340ca 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -49,6 +49,9 @@ Library Build ----- +- Issue #18256: Compilation fix for recent AIX releases. Patch by + David Edelsohn. + - Issue #18098: The deprecated OS X Build Applet.app fails to build on OS X 10.8 systems because the Apple-deprecated QuickDraw headers have been removed from Xcode 4. Skip building it in this case. diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index c1c92d1a15e..c9ed796cd0f 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -145,6 +145,7 @@ static void PyThread__init_thread(void) { #if defined(_AIX) && defined(__GNUC__) + extern void pthread_init(void); pthread_init(); #endif } @@ -394,6 +395,7 @@ PyThread_free_lock(PyThread_type_lock lock) pthread_lock *thelock = (pthread_lock *)lock; int status, error = 0; + (void) error; /* silence unused-but-set-variable warning */ dprintf(("PyThread_free_lock(%p) called\n", lock)); status = pthread_mutex_destroy( &thelock->mut ); @@ -445,6 +447,7 @@ PyThread_release_lock(PyThread_type_lock lock) pthread_lock *thelock = (pthread_lock *)lock; int status, error = 0; + (void) error; /* silence unused-but-set-variable warning */ dprintf(("PyThread_release_lock(%p) called\n", lock)); status = pthread_mutex_lock( &thelock->mut ); From d46aa331e1056e7864fe1ed861468b52042ec3ee Mon Sep 17 00:00:00 2001 From: R David Murray Date: Wed, 19 Jun 2013 17:00:43 -0400 Subject: [PATCH 170/480] Tweak programming faq examples so that it (mostly) passes doctest. Back port of 0113247f894b from 3.3. The exceptions are the import related questions at the end, which need to be rewritten anyway, and a math example that doesn't exist in the 3.3+ docs that I didn't bother trying to fix. --- Doc/faq/programming.rst | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 16fabe1ddf6..3878006a982 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -360,9 +360,9 @@ Why do lambdas defined in a loop with different values all return the same resul Assume you use a for loop to define a few different lambdas (or even plain functions), e.g.:: - squares = [] - for x in range(5): - squares.append(lambda: x**2) + >>> squares = [] + >>> for x in range(5): + ... squares.append(lambda: x**2) This gives you a list that contains 5 lambdas that calculate ``x**2``. You might expect that, when called, they would return, respectively, ``0``, ``1``, @@ -387,9 +387,9 @@ changing the value of ``x`` and see how the results of the lambdas change:: In order to avoid this, you need to save the values in variables local to the lambdas, so that they don't rely on the value of the global ``x``:: - squares = [] - for x in range(5): - squares.append(lambda n=x: n**2) + >>> squares = [] + >>> for x in range(5): + ... squares.append(lambda n=x: n**2) Here, ``n=x`` creates a new variable ``n`` local to the lambda and computed when the lambda is defined so that it has the same value that ``x`` had at @@ -748,11 +748,11 @@ Comma is not an operator in Python. Consider this session:: Since the comma is not an operator, but a separator between expressions the above is evaluated as if you had entered:: - >>> ("a" in "b"), "a" + ("a" in "b"), "a" not:: - >>> "a" in ("b", "a") + "a" in ("b", "a") The same is true of the various assignment operators (``=``, ``+=`` etc). They are not truly operators but syntactic delimiters in assignment statements. @@ -897,6 +897,7 @@ How do I modify a string in place? You can't, because strings are immutable. If you need an object with this ability, try converting the string to a list or use the array module:: + >>> import io >>> s = "Hello, world" >>> a = list(s) >>> print a @@ -910,7 +911,7 @@ ability, try converting the string to a list or use the array module:: >>> print a array('c', 'Hello, world') >>> a[0] = 'y' ; print a - array('c', 'yello world') + array('c', 'yello, world') >>> a.tostring() 'yello, world' @@ -1172,7 +1173,7 @@ How do I create a multidimensional list? You probably tried to make a multidimensional array like this:: - A = [[None] * 2] * 3 + >>> A = [[None] * 2] * 3 This looks correct if you print it:: @@ -1740,13 +1741,13 @@ file is automatic if you're importing a module and Python has the ability (permissions, free space, etc...) to write the compiled module back to the directory. -Running Python on a top level script is not considered an import and no ``.pyc`` -will be created. For example, if you have a top-level module ``abc.py`` that -imports another module ``xyz.py``, when you run abc, ``xyz.pyc`` will be created -since xyz is imported, but no ``abc.pyc`` file will be created since ``abc.py`` -isn't being imported. +Running Python on a top level script is not considered an import and no +``.pyc`` will be created. For example, if you have a top-level module +``foo.py`` that imports another module ``xyz.py``, when you run ``foo``, +``xyz.pyc`` will be created since ``xyz`` is imported, but no ``foo.pyc`` file +will be created since ``foo.py`` isn't being imported. -If you need to create abc.pyc -- that is, to create a .pyc file for a module +If you need to create ``foo.pyc`` -- that is, to create a ``.pyc`` file for a module that is not imported -- you can, using the :mod:`py_compile` and :mod:`compileall` modules. @@ -1754,9 +1755,9 @@ The :mod:`py_compile` module can manually compile any module. One way is to use the ``compile()`` function in that module interactively:: >>> import py_compile - >>> py_compile.compile('abc.py') + >>> py_compile.compile('foo.py') # doctest: +SKIP -This will write the ``.pyc`` to the same location as ``abc.py`` (or you can +This will write the ``.pyc`` to the same location as ``foo.py`` (or you can override that with the optional parameter ``cfile``). You can also automatically compile all files in a directory or directories using From dc8ee40c40cd3e1baecb6feae0c67b03db043544 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 22 Jun 2013 00:51:01 -0700 Subject: [PATCH 171/480] Arrange structure to match the common access patterns. --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 8a43d9a028a..f175fcaa0ea 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -48,8 +48,8 @@ typedef struct BLOCK { struct BLOCK *leftlink; - struct BLOCK *rightlink; PyObject *data[BLOCKLEN]; + struct BLOCK *rightlink; } block; #define MAXFREEBLOCKS 10 From bede9631cf1f8f6131f9a71135a73fd5bacd0acf Mon Sep 17 00:00:00 2001 From: Andrew Kuchling Date: Sat, 22 Jun 2013 12:33:05 -0400 Subject: [PATCH 172/480] #18113: avoid segfault if Py_XDECREF triggers code that calls set_panel_userptr again Problem noted & original patch by Serhiy Storchaka; I tweaked the patch a bit. --- Lib/test/test_curses.py | 9 +++++++++ Modules/_curses_panel.c | 11 ++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 387a1852d9d..fcf9618c5bc 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -262,6 +262,14 @@ def test_userptr_memory_leak(stdscr): if sys.getrefcount(obj) != nrefs: raise RuntimeError, "set_userptr leaked references" +def test_userptr_segfault(stdscr): + panel = curses.panel.new_panel(stdscr) + class A: + def __del__(self): + panel.set_userptr(None) + panel.set_userptr(A()) + panel.set_userptr(None) + def test_resize_term(stdscr): if hasattr(curses, 'resizeterm'): lines, cols = curses.LINES, curses.COLS @@ -281,6 +289,7 @@ def main(stdscr): window_funcs(stdscr) test_userptr_without_set(stdscr) test_userptr_memory_leak(stdscr) + test_userptr_segfault(stdscr) test_resize_term(stdscr) test_issue6243(stdscr) finally: diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index dd231e767fa..bdd5cf0177e 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -294,12 +294,17 @@ static PyObject * PyCursesPanel_set_panel_userptr(PyCursesPanelObject *self, PyObject *obj) { PyObject *oldobj; + int rc; PyCursesInitialised; + Py_INCREF(obj); oldobj = (PyObject *) panel_userptr(self->pan); + rc = set_panel_userptr(self->pan, (void*)obj); + if (rc == ERR) { + /* In case of an ncurses error, decref the new object again */ + Py_DECREF(obj); + } Py_XDECREF(oldobj); - Py_INCREF(obj); - return PyCursesCheckERR(set_panel_userptr(self->pan, (void*)obj), - "set_panel_userptr"); + return PyCursesCheckERR(rc, "set_panel_userptr"); } static PyObject * From db840036b7dd10c014d0c2c18bb00b5d497fc601 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 22 Jun 2013 18:58:51 +0200 Subject: [PATCH 173/480] transplant test_stat from default to 2.7 in order to make sure it works on all supported platforms --- Lib/test/test_stat.py | 171 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 Lib/test/test_stat.py diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py new file mode 100644 index 00000000000..ac11a0f5cc3 --- /dev/null +++ b/Lib/test/test_stat.py @@ -0,0 +1,171 @@ +import unittest +import os +from test.test_support import TESTFN, run_unittest +import stat + +class TestFilemode(unittest.TestCase): + file_flags = {'SF_APPEND', 'SF_ARCHIVED', 'SF_IMMUTABLE', 'SF_NOUNLINK', + 'SF_SNAPSHOT', 'UF_APPEND', 'UF_COMPRESSED', 'UF_HIDDEN', + 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE'} + + formats = {'S_IFBLK', 'S_IFCHR', 'S_IFDIR', 'S_IFIFO', 'S_IFLNK', + 'S_IFREG', 'S_IFSOCK'} + + format_funcs = {'S_ISBLK', 'S_ISCHR', 'S_ISDIR', 'S_ISFIFO', 'S_ISLNK', + 'S_ISREG', 'S_ISSOCK'} + + stat_struct = { + 'ST_MODE': 0, + 'ST_INO': 1, + 'ST_DEV': 2, + 'ST_NLINK': 3, + 'ST_UID': 4, + 'ST_GID': 5, + 'ST_SIZE': 6, + 'ST_ATIME': 7, + 'ST_MTIME': 8, + 'ST_CTIME': 9} + + # permission bit value are defined by POSIX + permission_bits = { + 'S_ISUID': 0o4000, + 'S_ISGID': 0o2000, + 'S_ENFMT': 0o2000, + 'S_ISVTX': 0o1000, + 'S_IRWXU': 0o700, + 'S_IRUSR': 0o400, + 'S_IREAD': 0o400, + 'S_IWUSR': 0o200, + 'S_IWRITE': 0o200, + 'S_IXUSR': 0o100, + 'S_IEXEC': 0o100, + 'S_IRWXG': 0o070, + 'S_IRGRP': 0o040, + 'S_IWGRP': 0o020, + 'S_IXGRP': 0o010, + 'S_IRWXO': 0o007, + 'S_IROTH': 0o004, + 'S_IWOTH': 0o002, + 'S_IXOTH': 0o001} + + def setUp(self): + try: + os.remove(TESTFN) + except OSError: + try: + os.rmdir(TESTFN) + except OSError: + pass + tearDown = setUp + + def get_mode(self, fname=TESTFN): + return os.lstat(fname).st_mode + + def assertS_IS(self, name, mode): + # test format, lstrip is for S_IFIFO + fmt = getattr(stat, "S_IF" + name.lstrip("F")) + self.assertEqual(stat.S_IFMT(mode), fmt) + # test that just one function returns true + testname = "S_IS" + name + for funcname in self.format_funcs: + func = getattr(stat, funcname, None) + if func is None: + if funcname == testname: + raise ValueError(funcname) + continue + if funcname == testname: + self.assertTrue(func(mode)) + else: + self.assertFalse(func(mode)) + + def test_mode(self): + with open(TESTFN, 'w'): + pass + if os.name == 'posix': + os.chmod(TESTFN, 0o700) + st_mode = self.get_mode() + self.assertS_IS("REG", st_mode) + self.assertEqual(stat.S_IMODE(st_mode), + stat.S_IRWXU) + + os.chmod(TESTFN, 0o070) + st_mode = self.get_mode() + self.assertS_IS("REG", st_mode) + self.assertEqual(stat.S_IMODE(st_mode), + stat.S_IRWXG) + + os.chmod(TESTFN, 0o007) + st_mode = self.get_mode() + self.assertS_IS("REG", st_mode) + self.assertEqual(stat.S_IMODE(st_mode), + stat.S_IRWXO) + + os.chmod(TESTFN, 0o444) + st_mode = self.get_mode() + self.assertS_IS("REG", st_mode) + self.assertEqual(stat.S_IMODE(st_mode), 0o444) + else: + os.chmod(TESTFN, 0o700) + st_mode = self.get_mode() + self.assertS_IS("REG", st_mode) + self.assertEqual(stat.S_IFMT(st_mode), + stat.S_IFREG) + + def test_directory(self): + os.mkdir(TESTFN) + os.chmod(TESTFN, 0o700) + st_mode = self.get_mode() + self.assertS_IS("DIR", st_mode) + + @unittest.skipUnless(hasattr(os, 'symlink'), 'os.symlink not available') + def test_link(self): + try: + os.symlink(os.getcwd(), TESTFN) + except (OSError, NotImplementedError) as err: + raise unittest.SkipTest(str(err)) + else: + st_mode = self.get_mode() + self.assertS_IS("LNK", st_mode) + + @unittest.skipUnless(hasattr(os, 'mkfifo'), 'os.mkfifo not available') + def test_fifo(self): + os.mkfifo(TESTFN, 0o700) + st_mode = self.get_mode() + self.assertS_IS("FIFO", st_mode) + + @unittest.skipUnless(os.name == 'posix', 'requires Posix') + def test_devices(self): + if os.path.exists(os.devnull): + st_mode = self.get_mode(os.devnull) + self.assertS_IS("CHR", st_mode) + # Linux block devices, BSD has no block devices anymore + for blockdev in ("/dev/sda", "/dev/hda"): + if os.path.exists(blockdev): + st_mode = self.get_mode(blockdev) + self.assertS_IS("BLK", st_mode) + break + + def test_module_attributes(self): + for key, value in self.stat_struct.items(): + modvalue = getattr(stat, key) + self.assertEqual(value, modvalue, key) + for key, value in self.permission_bits.items(): + modvalue = getattr(stat, key) + self.assertEqual(value, modvalue, key) + for key in self.file_flags: + modvalue = getattr(stat, key) + self.assertIsInstance(modvalue, int) + for key in self.formats: + modvalue = getattr(stat, key) + self.assertIsInstance(modvalue, int) + for key in self.format_funcs: + func = getattr(stat, key) + self.assertTrue(callable(func)) + self.assertEqual(func(0), 0) + + +def test_main(): + run_unittest(TestFilemode) + +if __name__ == '__main__': + test_main() From 5209236a5b61eaf8df8e9b1fea8997206d579498 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 22 Jun 2013 11:16:36 -0700 Subject: [PATCH 174/480] backout 5accb0ac8bfb; needs more discussion on python-dev --- Lib/test/test_deque.py | 2 +- Modules/_collectionsmodule.c | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 98b203edb60..595a0c4a35c 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -522,7 +522,7 @@ class C(object): @test_support.cpython_only def test_sizeof(self): - BLOCKLEN = 64 + BLOCKLEN = 62 basesize = test_support.calcobjsize('2P4PlP') blocksize = struct.calcsize('2P%dP' % BLOCKLEN) self.assertEqual(object.__sizeof__(deque()), basesize) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index f175fcaa0ea..abd80e06c43 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -8,13 +8,12 @@ */ /* The block length may be set to any number over 1. Larger numbers - * reduce the number of calls to the memory allocator, give faster - * indexing and rotation, and reduce the link::data overhead ratio. - * Ideally, the block length should be a power-of-two for faster - * division/modulo computations during indexing. + * reduce the number of calls to the memory allocator but take more + * memory. Ideally, BLOCKLEN should be set with an eye to the + * length of a cache line. */ -#define BLOCKLEN 64 +#define BLOCKLEN 62 #define CENTER ((BLOCKLEN - 1) / 2) /* A `dequeobject` is composed of a doubly-linked list of `block` nodes. @@ -59,8 +58,13 @@ static block *freeblocks[MAXFREEBLOCKS]; static block * newblock(block *leftlink, block *rightlink, Py_ssize_t len) { block *b; - /* To prevent len from overflowing PY_SSIZE_T_MAX on 32-bit machines, we - * refuse to allocate new blocks if the current len is nearing overflow. */ + /* To prevent len from overflowing PY_SSIZE_T_MAX on 64-bit machines, we + * refuse to allocate new blocks if the current len is dangerously + * close. There is some extra margin to prevent spurious arithmetic + * overflows at various places. The following check ensures that + * the blocks allocated to the deque, in the worst case, can only + * have PY_SSIZE_T_MAX-2 entries in total. + */ if (len >= PY_SSIZE_T_MAX - 2*BLOCKLEN) { PyErr_SetString(PyExc_OverflowError, "cannot add more blocks to the deque"); From d240cb2a0ea8bddec5f0828d277fe5852c2830f3 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 22 Jun 2013 18:26:38 -0400 Subject: [PATCH 175/480] #18151, part 2: Silence debug build resource warning for each file opened by 'Find in files' by replacing 'open with implicit close' by 'with open' in GrepDialog method grep_it. Streamline code with enumerate(), direct file iteration, and output tweak. Add test for this method, including output format. --- Lib/idlelib/GrepDialog.py | 42 +++++++-------- Lib/idlelib/idle_test/test_grep.py | 82 ++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 24 deletions(-) create mode 100644 Lib/idlelib/idle_test/test_grep.py diff --git a/Lib/idlelib/GrepDialog.py b/Lib/idlelib/GrepDialog.py index e40e5468c0f..c8b703c3ac8 100644 --- a/Lib/idlelib/GrepDialog.py +++ b/Lib/idlelib/GrepDialog.py @@ -81,31 +81,19 @@ def grep_it(self, prog, path): hits = 0 for fn in list: try: - f = open(fn) - except IOError, msg: + with open(fn) as f: + for lineno, line in enumerate(f, 1): + if line[-1:] == '\n': + line = line[:-1] + if prog.search(line): + sys.stdout.write("%s: %s: %s\n" % + (fn, lineno, line)) + hits += 1 + except IOError as msg: print msg - continue - lineno = 0 - while 1: - block = f.readlines(100000) - if not block: - break - for line in block: - lineno = lineno + 1 - if line[-1:] == '\n': - line = line[:-1] - if prog.search(line): - sys.stdout.write("%s: %s: %s\n" % (fn, lineno, line)) - hits = hits + 1 - if hits: - if hits == 1: - s = "" - else: - s = "s" - print "Found", hits, "hit%s." % s - print "(Hint: right-click to open locations.)" - else: - print "No hits." + print(("Hits found: %s\n" + "(Hint: right-click to open locations.)" + % hits) if hits else "No hits.") def findfiles(self, dir, base, rec): try: @@ -131,3 +119,9 @@ def close(self, event=None): if self.top: self.top.grab_release() self.top.withdraw() + +if __name__ == "__main__": + # A human test is a bit tricky since EditorWindow() imports this module. + # Hence Idle must be restarted after editing this file for a live test. + import unittest + unittest.main('idlelib.idle_test.test_grep', verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_grep.py b/Lib/idlelib/idle_test/test_grep.py new file mode 100644 index 00000000000..e9f4f22ae65 --- /dev/null +++ b/Lib/idlelib/idle_test/test_grep.py @@ -0,0 +1,82 @@ +""" !Changing this line will break Test_findfile.test_found! +Non-gui unit tests for idlelib.GrepDialog methods. +dummy_command calls grep_it calls findfiles. +An exception raised in one method will fail callers. +Otherwise, tests are mostly independent. +*** Currently only test grep_it. +""" +import unittest +from test.test_support import captured_stdout, findfile +from idlelib.idle_test.mock_tk import Var +from idlelib.GrepDialog import GrepDialog +import re + +__file__ = findfile('idlelib/idle_test') + '/test_grep.py' + +class Dummy_searchengine: + '''GrepDialog.__init__ calls parent SearchDiabolBase which attaches the + passed in SearchEngine instance as attribute 'engine'. Only a few of the + many possible self.engine.x attributes are needed here. + ''' + def getpat(self): + return self._pat + +searchengine = Dummy_searchengine() + +class Dummy_grep: + # Methods tested + #default_command = GrepDialog.default_command + grep_it = GrepDialog.grep_it.im_func + findfiles = GrepDialog.findfiles.im_func + # Other stuff needed + recvar = Var(False) + engine = searchengine + def close(self): # gui method + pass + +grep = Dummy_grep() + +class FindfilesTest(unittest.TestCase): + # findfiles is really a function, not a method, could be iterator + # test that filename return filename + # test that idlelib has many .py files + # test that recursive flag adds idle_test .py files + pass + +class Grep_itTest(unittest.TestCase): + # Test captured reports with 0 and some hits. + # Should test file names, but Windows reports have mixed / and \ separators + # from incomplete replacement, so 'later'. + + def report(self, pat): + grep.engine._pat = pat + with captured_stdout() as s: + grep.grep_it(re.compile(pat), __file__) + lines = s.getvalue().split('\n') + lines.pop() # remove bogus '' after last \n + return lines + + def test_unfound(self): + pat = 'xyz*'*7 + lines = self.report(pat) + self.assertEqual(len(lines), 2) + self.assertIn(pat, lines[0]) + self.assertEqual(lines[1], 'No hits.') + + def test_found(self): + + pat = '""" !Changing this line will break Test_findfile.test_found!' + lines = self.report(pat) + self.assertEqual(len(lines), 5) + self.assertIn(pat, lines[0]) + self.assertIn('py: 1:', lines[1]) # line number 1 + self.assertIn('2', lines[3]) # hits found 2 + self.assertTrue(lines[4].startswith('(Hint:')) + +class Default_commandTest(unittest.TestCase): + # To write this, mode OutputWindow import to top of GrepDialog + # so it can be replaced by captured_stdout in class setup/teardown. + pass + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) From ff7eec63c21d0e836c2e76e9a32bc196cfaa4e32 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 23 Jun 2013 14:56:57 +0200 Subject: [PATCH 176/480] Issue #18137: Detect integer overflow on precision in float.__format__() and complex.__format__(). --- Lib/test/test_format.py | 17 +++++++++++++++++ Misc/NEWS | 3 +++ Objects/stringlib/formatter.h | 16 ++++++++++++++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index dd30efabb6c..0ad8b02d7b5 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -302,6 +302,23 @@ def __oct__(self): def test_main(): test_support.run_unittest(FormatTest) + def test_precision(self): + INT_MAX = 2147483647 + + f = 1.2 + self.assertEqual(format(f, ".0f"), "1") + self.assertEqual(format(f, ".3f"), "1.200") + with self.assertRaises(ValueError) as cm: + format(f, ".%sf" % (INT_MAX + 1)) + self.assertEqual(str(cm.exception), "precision too big") + + c = complex(f) + self.assertEqual(format(f, ".0f"), "1") + self.assertEqual(format(f, ".3f"), "1.200") + with self.assertRaises(ValueError) as cm: + format(f, ".%sf" % (INT_MAX + 1)) + self.assertEqual(str(cm.exception), "precision too big") + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 941818340ca..b4553668959 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -9,6 +9,9 @@ What's New in Python 2.7.6? Core and Builtins ----------------- +- Issue #18137: Detect integer overflow on precision in float.__format__() + and complex.__format__(). + - Issue #18038: SyntaxError raised during compilation sources with illegal encoding now always contains an encoding name. diff --git a/Objects/stringlib/formatter.h b/Objects/stringlib/formatter.h index 6b282249b17..fd227511f36 100644 --- a/Objects/stringlib/formatter.h +++ b/Objects/stringlib/formatter.h @@ -928,7 +928,7 @@ format_float_internal(PyObject *value, Py_ssize_t n_total; int has_decimal; double val; - Py_ssize_t precision = format->precision; + Py_ssize_t precision; Py_ssize_t default_precision = 6; STRINGLIB_CHAR type = format->type; int add_pct = 0; @@ -947,6 +947,12 @@ format_float_internal(PyObject *value, from a hard-code pseudo-locale */ LocaleInfo locale; + if (format->precision > INT_MAX) { + PyErr_SetString(PyExc_ValueError, "precision too big"); + goto done; + } + precision = (int)format->precision; + /* Alternate is not allowed on floats. */ if (format->alternate) { PyErr_SetString(PyExc_ValueError, @@ -1078,7 +1084,7 @@ format_complex_internal(PyObject *value, Py_ssize_t n_im_total; int re_has_decimal; int im_has_decimal; - Py_ssize_t precision = format->precision; + Py_ssize_t precision; Py_ssize_t default_precision = 6; STRINGLIB_CHAR type = format->type; STRINGLIB_CHAR *p_re; @@ -1107,6 +1113,12 @@ format_complex_internal(PyObject *value, from a hard-code pseudo-locale */ LocaleInfo locale; + if (format->precision > INT_MAX) { + PyErr_SetString(PyExc_ValueError, "precision too big"); + goto done; + } + precision = (int)format->precision; + /* Alternate is not allowed on complex. */ if (format->alternate) { PyErr_SetString(PyExc_ValueError, From 219a011af741f36f92ce2aebb5c1d85fa0d547ad Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 23 Jun 2013 15:15:10 +0200 Subject: [PATCH 177/480] Issue #18135: Fix a possible integer overflow in ssl.SSLSocket.write() for strings longer than 2 gigabytes. --- Misc/NEWS | 3 +++ Modules/_ssl.c | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index b4553668959..d7e15f38780 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,9 @@ Core and Builtins Library ------- +- Issue #18135: Fix a possible integer overflow in ssl.SSLSocket.write() + for strings longer than 2 gigabytes. + - Issue #18167: cgi.FieldStorage no more fails to handle multipart/form-data when \r\n appears at end of 65535 bytes without other newlines. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 195e5b6491f..907429de160 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1212,8 +1212,13 @@ static PyObject *PySSL_SSLwrite(PySSLObject *self, PyObject *args) goto error; } do { + if (buf.len <= INT_MAX) + len = (int)buf.len; + else + len = INT_MAX; + PySSL_BEGIN_ALLOW_THREADS - len = SSL_write(self->ssl, buf.buf, buf.len); + len = SSL_write(self->ssl, buf.buf, len); err = SSL_get_error(self->ssl, len); PySSL_END_ALLOW_THREADS if (PyErr_CheckSignals()) { From 6fedd5a634532b7e15b3c2a6a0b3cfda884910c6 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 23 Jun 2013 16:27:01 +0200 Subject: [PATCH 178/480] Solaris' /dev/null is a symlink. The device test now uses stat instead of lstat to compensate for symlinks. --- Lib/test/test_stat.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index ac11a0f5cc3..a71f599b8d6 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -58,8 +58,12 @@ def setUp(self): pass tearDown = setUp - def get_mode(self, fname=TESTFN): - return os.lstat(fname).st_mode + def get_mode(self, fname=TESTFN, lstat=True): + if lstat: + st_mode = os.lstat(fname).st_mode + else: + st_mode = os.stat(fname).st_mode + return st_mode def assertS_IS(self, name, mode): # test format, lstrip is for S_IFIFO @@ -136,12 +140,12 @@ def test_fifo(self): @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_devices(self): if os.path.exists(os.devnull): - st_mode = self.get_mode(os.devnull) + st_mode = self.get_mode(os.devnull, lstat=False) self.assertS_IS("CHR", st_mode) # Linux block devices, BSD has no block devices anymore for blockdev in ("/dev/sda", "/dev/hda"): if os.path.exists(blockdev): - st_mode = self.get_mode(blockdev) + st_mode = self.get_mode(blockdev, lstat=False) self.assertS_IS("BLK", st_mode) break From c304d7ca1a114fee8e8d3ef7990c60b0b01e4dd4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 23 Jun 2013 20:22:09 +0300 Subject: [PATCH 179/480] Issue #18184: PyUnicode_FromFormat() and PyUnicode_FromFormatV() now raise OverflowError when an argument of %c format is out of range. --- Misc/NEWS | 3 +++ Objects/unicodeobject.c | 19 ++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index d7e15f38780..09d252a1174 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -9,6 +9,9 @@ What's New in Python 2.7.6? Core and Builtins ----------------- +- Issue #18184: PyUnicode_FromFormat() and PyUnicode_FromFormatV() now raise + OverflowError when an argument of %c format is out of range. + - Issue #18137: Detect integer overflow on precision in float.__format__() and complex.__format__(). diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 0ead06f242c..64a5ef557c0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -740,8 +740,25 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) switch (*f) { case 'c': - (void)va_arg(count, int); + { + int ordinal = va_arg(count, int); +#ifdef Py_UNICODE_WIDE + if (ordinal < 0 || ordinal > 0x10ffff) { + PyErr_SetString(PyExc_OverflowError, + "%c arg not in range(0x110000) " + "(wide Python build)"); + goto fail; + } +#else + if (ordinal < 0 || ordinal > 0xffff) { + PyErr_SetString(PyExc_OverflowError, + "%c arg not in range(0x10000) " + "(narrow Python build)"); + goto fail; + } +#endif /* fall through... */ + } case '%': n++; break; From 59ffd0028e3f6bab4d77bd22f4b717df81cf91d8 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 23 Jun 2013 11:38:11 -0700 Subject: [PATCH 180/480] also backout f1dc30a1be72 for not being a bugfix --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index abd80e06c43..371631ce266 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -47,8 +47,8 @@ typedef struct BLOCK { struct BLOCK *leftlink; - PyObject *data[BLOCKLEN]; struct BLOCK *rightlink; + PyObject *data[BLOCKLEN]; } block; #define MAXFREEBLOCKS 10 From c26fe82f72cdfeed607020271ce827d4a7dc5cb5 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sun, 23 Jun 2013 16:02:34 -0400 Subject: [PATCH 181/480] #18179: document the local_hostname parameter. Original patch by Berker Peksag. --- Doc/library/smtplib.rst | 12 +++++++++--- Lib/smtplib.py | 9 ++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 18560c129f7..27d722d96a7 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -25,7 +25,10 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). A :class:`SMTP` instance encapsulates an SMTP connection. It has methods that support a full repertoire of SMTP and ESMTP operations. If the optional host and port parameters are given, the SMTP :meth:`connect` method is called - with those parameters during initialization. If the :meth:`connect` call + with those parameters during initialization. If specified, *local_hostname* is + used as the FQDN of the local host in the HELO/EHLO command. Otherwise, the + local hostname is found using :func:`socket.getfqdn`. If the + :meth:`connect` call returns anything other than a success code, an :exc:`SMTPConnectError` is raised. The optional *timeout* parameter specifies a timeout in seconds for blocking operations like the connection attempt (if not specified, the @@ -45,7 +48,9 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is required from the beginning of the connection and using :meth:`starttls` is not appropriate. If *host* is not specified, the local host is used. If - *port* is omitted, the standard SMTP-over-SSL port (465) is used. *keyfile* + *port* is omitted, the standard SMTP-over-SSL port (465) is used. + *local_hostname* has the same meaning as it does for the :class:`SMTP` class. + *keyfile* and *certfile* are also optional, and can contain a PEM formatted private key and certificate chain file for the SSL connection. The optional *timeout* parameter specifies a timeout in seconds for blocking operations like the @@ -59,7 +64,8 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). The LMTP protocol, which is very similar to ESMTP, is heavily based on the standard SMTP client. It's common to use Unix sockets for LMTP, so our :meth:`connect` - method must support that as well as a regular host:port server. To specify a + method must support that as well as a regular host:port server. *local_hostname* + has the same meaning as it does for the :class:`SMTP` class. To specify a Unix socket, you must use an absolute path for *host*, starting with a '/'. Authentication is supported, using the regular SMTP mechanism. When using a Unix diff --git a/Lib/smtplib.py b/Lib/smtplib.py index a3213b33d71..2c8dc42d9eb 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -240,7 +240,8 @@ def __init__(self, host='', port=0, local_hostname=None, By default, smtplib.SMTP_PORT is used. If a host is specified the connect method is called, and if it returns anything other than a success code an SMTPConnectError is raised. If specified, - `local_hostname` is used as the FQDN of the local host. By default, + `local_hostname` is used as the FQDN of the local host for the + HELO/EHLO command. Otherwise, the local hostname is found using socket.getfqdn(). """ @@ -762,7 +763,8 @@ class SMTP_SSL(SMTP): """ This is a subclass derived from SMTP that connects over an SSL encrypted socket (to use this class you need a socket module that was compiled with SSL support). If host is not specified, '' (the local host) is used. If port is - omitted, the standard SMTP-over-SSL port (465) is used. keyfile and certfile + omitted, the standard SMTP-over-SSL port (465) is used. local_hostname + has the same meaning as it does in the SMTP class. keyfile and certfile are also optional - they can contain a PEM formatted private key and certificate chain file for the SSL connection. """ @@ -797,7 +799,8 @@ class LMTP(SMTP): The LMTP protocol, which is very similar to ESMTP, is heavily based on the standard SMTP client. It's common to use Unix sockets for LMTP, so our connect() method must support that as well as a regular - host:port server. To specify a Unix socket, you must use an absolute + host:port server. local_hostname has the same meaning as it does in the + SMTP class. To specify a Unix socket, you must use an absolute path as the host, starting with a '/'. Authentication is supported, using the regular SMTP mechanism. When From d951e5674ffca646813d8f2ff94d17bf1b9ad823 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sun, 23 Jun 2013 16:10:37 -0400 Subject: [PATCH 182/480] #18179: reflow paragraphs. --- Doc/library/smtplib.rst | 48 ++++++++++++++++++++--------------------- Lib/smtplib.py | 32 ++++++++++++++------------- 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 27d722d96a7..4f3b572aab4 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -24,15 +24,15 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). A :class:`SMTP` instance encapsulates an SMTP connection. It has methods that support a full repertoire of SMTP and ESMTP operations. If the optional - host and port parameters are given, the SMTP :meth:`connect` method is called - with those parameters during initialization. If specified, *local_hostname* is - used as the FQDN of the local host in the HELO/EHLO command. Otherwise, the - local hostname is found using :func:`socket.getfqdn`. If the - :meth:`connect` call - returns anything other than a success code, an :exc:`SMTPConnectError` is - raised. The optional *timeout* parameter specifies a timeout in seconds for - blocking operations like the connection attempt (if not specified, the - global default timeout setting will be used). + host and port parameters are given, the SMTP :meth:`connect` method is + called with those parameters during initialization. If specified, + *local_hostname* is used as the FQDN of the local host in the HELO/EHLO + command. Otherwise, the local hostname is found using + :func:`socket.getfqdn`. If the :meth:`connect` call returns anything other + than a success code, an :exc:`SMTPConnectError` is raised. The optional + *timeout* parameter specifies a timeout in seconds for blocking operations + like the connection attempt (if not specified, the global default timeout + setting will be used). For normal use, you should only require the initialization/connect, :meth:`sendmail`, and :meth:`~smtplib.quit` methods. @@ -49,13 +49,12 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). required from the beginning of the connection and using :meth:`starttls` is not appropriate. If *host* is not specified, the local host is used. If *port* is omitted, the standard SMTP-over-SSL port (465) is used. - *local_hostname* has the same meaning as it does for the :class:`SMTP` class. - *keyfile* - and *certfile* are also optional, and can contain a PEM formatted private key - and certificate chain file for the SSL connection. The optional *timeout* - parameter specifies a timeout in seconds for blocking operations like the - connection attempt (if not specified, the global default timeout setting - will be used). + *local_hostname* has the same meaning as it does for the :class:`SMTP` + class. *keyfile* and *certfile* are also optional, and can contain a PEM + formatted private key and certificate chain file for the SSL connection. The + optional *timeout* parameter specifies a timeout in seconds for blocking + operations like the connection attempt (if not specified, the global default + timeout setting will be used). .. versionadded:: 2.6 @@ -63,14 +62,15 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). .. class:: LMTP([host[, port[, local_hostname]]]) The LMTP protocol, which is very similar to ESMTP, is heavily based on the - standard SMTP client. It's common to use Unix sockets for LMTP, so our :meth:`connect` - method must support that as well as a regular host:port server. *local_hostname* - has the same meaning as it does for the :class:`SMTP` class. To specify a - Unix socket, you must use an absolute path for *host*, starting with a '/'. - - Authentication is supported, using the regular SMTP mechanism. When using a Unix - socket, LMTP generally don't support or require any authentication, but your - mileage might vary. + standard SMTP client. It's common to use Unix sockets for LMTP, so our + :meth:`connect` method must support that as well as a regular host:port + server. *local_hostname* has the same meaning as it does for the + :class:`SMTP` class. To specify a Unix socket, you must use an absolute + path for *host*, starting with a '/'. + + Authentication is supported, using the regular SMTP mechanism. When using a + Unix socket, LMTP generally don't support or require any authentication, but + your mileage might vary. .. versionadded:: 2.6 diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 2c8dc42d9eb..7f07840044b 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -238,11 +238,11 @@ def __init__(self, host='', port=0, local_hostname=None, If specified, `host' is the name of the remote host to which to connect. If specified, `port' specifies the port to which to connect. By default, smtplib.SMTP_PORT is used. If a host is specified the - connect method is called, and if it returns anything other than - a success code an SMTPConnectError is raised. If specified, + connect method is called, and if it returns anything other than a + success code an SMTPConnectError is raised. If specified, `local_hostname` is used as the FQDN of the local host for the - HELO/EHLO command. Otherwise, - the local hostname is found using socket.getfqdn(). + HELO/EHLO command. Otherwise, the local hostname is found using + socket.getfqdn(). """ self.timeout = timeout @@ -760,13 +760,15 @@ def quit(self): if _have_ssl: class SMTP_SSL(SMTP): - """ This is a subclass derived from SMTP that connects over an SSL encrypted - socket (to use this class you need a socket module that was compiled with SSL - support). If host is not specified, '' (the local host) is used. If port is - omitted, the standard SMTP-over-SSL port (465) is used. local_hostname - has the same meaning as it does in the SMTP class. keyfile and certfile - are also optional - they can contain a PEM formatted private key and - certificate chain file for the SSL connection. + """ This is a subclass derived from SMTP that connects over an SSL + encrypted socket (to use this class you need a socket module that was + compiled with SSL support). If host is not specified, '' (the local + host) is used. If port is omitted, the standard SMTP-over-SSL port + (465) is used. local_hostname has the same meaning as it does in the + SMTP class. keyfile and certfile are also optional - they can contain + a PEM formatted private key and certificate chain file for the SSL + connection. + """ default_port = SMTP_SSL_PORT @@ -797,10 +799,10 @@ class LMTP(SMTP): """LMTP - Local Mail Transfer Protocol The LMTP protocol, which is very similar to ESMTP, is heavily based - on the standard SMTP client. It's common to use Unix sockets for LMTP, - so our connect() method must support that as well as a regular - host:port server. local_hostname has the same meaning as it does in the - SMTP class. To specify a Unix socket, you must use an absolute + on the standard SMTP client. It's common to use Unix sockets for + LMTP, so our connect() method must support that as well as a regular + host:port server. local_hostname has the same meaning as it does in + the SMTP class. To specify a Unix socket, you must use an absolute path as the host, starting with a '/'. Authentication is supported, using the regular SMTP mechanism. When From 9bcd4d83f1e0acf01006adce1d139cb1fd136cc7 Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Mon, 24 Jun 2013 14:45:24 +0100 Subject: [PATCH 183/480] Issue #18277: Document quirks of multiprocessing queue. --- Doc/library/multiprocessing.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 6f9298ad28d..8904cc36474 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -486,6 +486,23 @@ Note that one can also create a shared queue by using a manager object -- see the :mod:`multiprocessing` namespace so you need to import them from :mod:`Queue`. +.. note:: + + When an object is put on a queue, the object is pickled and a + background thread later flushes the pickled data to an underlying + pipe. This has some consequences which are a little surprising, + but should not cause any pratical difficulties -- you can always + use a managed queue if they really bother you. + + (1) After putting an object on an empty queue there may be an + infinitessimal delay before the queue's :meth:`~Queue.empty` + method returns :const:`False` and :meth:`~Queue.get_nowait` can + return without raising :exc:`Queue.Empty`. + + (2) If multiple processes are enqueuing objects, it is possible for + the objects to be received at the other end out-of-order. + However, objects enqueued by the same process will always be in + the expected order with respect to each other. .. warning:: From 73815fee4daa5bbcdf15a9c088eb417bb6e00ee4 Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Mon, 24 Jun 2013 15:41:36 +0100 Subject: [PATCH 184/480] Issue #15818: Typo in docs. --- Doc/library/multiprocessing.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 8904cc36474..a7dd0b801bb 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -423,9 +423,9 @@ The :mod:`multiprocessing` package mostly replicates the API of the acquired a lock or semaphore etc. then terminating it is liable to cause other processes to deadlock. - Note that the :meth:`start`, :meth:`join`, :meth:`is_alive` and - :attr:`exit_code` methods should only be called by the process that created - the process object. + Note that the :meth:`start`, :meth:`join`, :meth:`is_alive`, + :meth:`terminate` and :attr:`exitcode` methods should only be called by + the process that created the process object. Example usage of some of the methods of :class:`Process`: From 0a9abde6bfd5cd5e7737028d1709a9a9f787369b Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Mon, 24 Jun 2013 18:11:21 +0100 Subject: [PATCH 185/480] Clarify note and fix typo. --- Doc/library/multiprocessing.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index a7dd0b801bb..bace92ac7cd 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -491,8 +491,9 @@ Note that one can also create a shared queue by using a manager object -- see When an object is put on a queue, the object is pickled and a background thread later flushes the pickled data to an underlying pipe. This has some consequences which are a little surprising, - but should not cause any pratical difficulties -- you can always - use a managed queue if they really bother you. + but should not cause any practical difficulties -- if they really + bother you then you can instead use a queue created with a + :ref:`manager `. (1) After putting an object on an empty queue there may be an infinitessimal delay before the queue's :meth:`~Queue.empty` From b9656d28d1e8c66f3edf5bb259b0740cfec1e5ef Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Mon, 24 Jun 2013 20:38:22 +0100 Subject: [PATCH 186/480] Fix typo. --- Doc/library/multiprocessing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index bace92ac7cd..3f368a316a4 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -496,7 +496,7 @@ Note that one can also create a shared queue by using a manager object -- see :ref:`manager `. (1) After putting an object on an empty queue there may be an - infinitessimal delay before the queue's :meth:`~Queue.empty` + infinitesimal delay before the queue's :meth:`~Queue.empty` method returns :const:`False` and :meth:`~Queue.get_nowait` can return without raising :exc:`Queue.Empty`. From 165e96f848cacbc616e4d587506ee5c09162782a Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 24 Jun 2013 14:21:43 -0700 Subject: [PATCH 187/480] Issue #18164: Backport the more detailed embedding compile-and-link section from the Python 3 documentation. --- Doc/extending/embedding.rst | 74 +++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index ae5efc40876..981e1d546e6 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -258,37 +258,55 @@ program. There is no need to recompile Python itself using C++. .. _link-reqs: -Linking Requirements -==================== - -While the :program:`configure` script shipped with the Python sources will -correctly build Python to export the symbols needed by dynamically linked -extensions, this is not automatically inherited by applications which embed the -Python library statically, at least on Unix. This is an issue when the -application is linked to the static runtime library (:file:`libpython.a`) and -needs to load dynamic extensions (implemented as :file:`.so` files). - -The problem is that some entry points are defined by the Python runtime solely -for extension modules to use. If the embedding application does not use any of -these entry points, some linkers will not include those entries in the symbol -table of the finished executable. Some additional options are needed to inform -the linker not to remove these symbols. - -Determining the right options to use for any given platform can be quite -difficult, but fortunately the Python configuration already has those values. -To retrieve them from an installed Python interpreter, start an interactive -interpreter and have a short session like this +Compiling and Linking under Unix-like systems +============================================= + +It is not necessarily trivial to find the right flags to pass to your +compiler (and linker) in order to embed the Python interpreter into your +application, particularly because Python needs to load library modules +implemented as C dynamic extensions (:file:`.so` files) linked against +it. + +To find out the required compiler and linker flags, you can execute the +:file:`python{X.Y}-config` script which is generated as part of the +installation process (a :file:`python-config` script may also be +available). This script has several options, of which the following will +be directly useful to you: + +* ``pythonX.Y-config --cflags`` will give you the recommended flags when + compiling:: + + $ /opt/bin/python2.7-config --cflags + -I/opt/include/python2.7 -fno-strict-aliasing -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes + +* ``pythonX.Y-config --ldflags`` will give you the recommended flags when + linking:: + + $ /opt/bin/python2.7-config --ldflags + -L/opt/lib/python2.7/config -lpthread -ldl -lutil -lm -lpython2.7 -Xlinker -export-dynamic + +.. note:: + To avoid confusion between several Python installations (and especially + between the system Python and your own compiled Python), it is recommended + that you use the absolute path to :file:`python{X.Y}-config`, as in the above + example. + +If this procedure doesn't work for you (it is not guaranteed to work for +all Unix-like platforms; however, we welcome :ref:`bug reports `) +you will have to read your system's documentation about dynamic linking and/or +examine Python's :file:`Makefile` (use :func:`sysconfig.get_makefile_filename` +to find its location) and compilation +options. In this case, the :mod:`sysconfig` module is a useful tool to +programmatically extract the configuration values that you will want to +combine together. For example: .. code-block:: python - >>> import distutils.sysconfig - >>> distutils.sysconfig.get_config_var('LINKFORSHARED') + >>> import sysconfig + >>> sysconfig.get_config_var('LIBS') + '-lpthread -ldl -lutil' + >>> sysconfig.get_config_var('LINKFORSHARED') '-Xlinker -export-dynamic' -.. index:: module: distutils.sysconfig - -The contents of the string presented will be the options that should be used. -If the string is empty, there's no need to add any additional options. The -:const:`LINKFORSHARED` definition corresponds to the variable of the same name -in Python's top-level :file:`Makefile`. +.. XXX similar documentation for Windows missing From 92cdd61f367f4b85a60d2f4a9a3bc69f723c6173 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 25 Jun 2013 00:48:02 +0200 Subject: [PATCH 188/480] Issue #18135: ssl.SSLSocket.write() now raises an OverflowError if the input string in longer than 2 gigabytes. The ssl module does not support partial write. --- Misc/NEWS | 5 +++-- Modules/_ssl.c | 13 +++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 09d252a1174..fff0032c268 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,8 +24,9 @@ Core and Builtins Library ------- -- Issue #18135: Fix a possible integer overflow in ssl.SSLSocket.write() - for strings longer than 2 gigabytes. +- Issue #18135: ssl.SSLSocket.write() now raises an OverflowError if the input + string in longer than 2 gigabytes. The ssl module does not support partial + write. - Issue #18167: cgi.FieldStorage no more fails to handle multipart/form-data when \r\n appears at end of 65535 bytes without other newlines. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 907429de160..afcc0174ab8 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1192,6 +1192,12 @@ static PyObject *PySSL_SSLwrite(PySSLObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "s*:write", &buf)) return NULL; + if (buf.len > INT_MAX) { + PyErr_Format(PyExc_OverflowError, + "string longer than %d bytes", INT_MAX); + goto error; + } + /* just in case the blocking state of the socket has been changed */ nonblocking = (self->Socket->sock_timeout >= 0.0); BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); @@ -1212,13 +1218,8 @@ static PyObject *PySSL_SSLwrite(PySSLObject *self, PyObject *args) goto error; } do { - if (buf.len <= INT_MAX) - len = (int)buf.len; - else - len = INT_MAX; - PySSL_BEGIN_ALLOW_THREADS - len = SSL_write(self->ssl, buf.buf, len); + len = SSL_write(self->ssl, buf.buf, (int)buf.len); err = SSL_get_error(self->ssl, len); PySSL_END_ALLOW_THREADS if (PyErr_CheckSignals()) { From feba89519c08ceb2309351fddbe9556e145ada00 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 25 Jun 2013 11:25:26 -0700 Subject: [PATCH 189/480] reapply f1dc30a1be72 --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 371631ce266..abd80e06c43 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -47,8 +47,8 @@ typedef struct BLOCK { struct BLOCK *leftlink; - struct BLOCK *rightlink; PyObject *data[BLOCKLEN]; + struct BLOCK *rightlink; } block; #define MAXFREEBLOCKS 10 From 704f49ddce2e292d1865d3558dd33bcd7414449b Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 25 Jun 2013 11:26:20 -0700 Subject: [PATCH 190/480] reapply f1dc30a1be72 --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index abd80e06c43..371631ce266 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -47,8 +47,8 @@ typedef struct BLOCK { struct BLOCK *leftlink; - PyObject *data[BLOCKLEN]; struct BLOCK *rightlink; + PyObject *data[BLOCKLEN]; } block; #define MAXFREEBLOCKS 10 From b605092fbe772e61162533ce87a167163251e05b Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 25 Jun 2013 11:34:48 -0700 Subject: [PATCH 191/480] reapply 5accb0ac8bfb --- Lib/test/test_deque.py | 2 +- Modules/_collectionsmodule.c | 18 +++++++----------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 595a0c4a35c..98b203edb60 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -522,7 +522,7 @@ class C(object): @test_support.cpython_only def test_sizeof(self): - BLOCKLEN = 62 + BLOCKLEN = 64 basesize = test_support.calcobjsize('2P4PlP') blocksize = struct.calcsize('2P%dP' % BLOCKLEN) self.assertEqual(object.__sizeof__(deque()), basesize) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 371631ce266..8a43d9a028a 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -8,12 +8,13 @@ */ /* The block length may be set to any number over 1. Larger numbers - * reduce the number of calls to the memory allocator but take more - * memory. Ideally, BLOCKLEN should be set with an eye to the - * length of a cache line. + * reduce the number of calls to the memory allocator, give faster + * indexing and rotation, and reduce the link::data overhead ratio. + * Ideally, the block length should be a power-of-two for faster + * division/modulo computations during indexing. */ -#define BLOCKLEN 62 +#define BLOCKLEN 64 #define CENTER ((BLOCKLEN - 1) / 2) /* A `dequeobject` is composed of a doubly-linked list of `block` nodes. @@ -58,13 +59,8 @@ static block *freeblocks[MAXFREEBLOCKS]; static block * newblock(block *leftlink, block *rightlink, Py_ssize_t len) { block *b; - /* To prevent len from overflowing PY_SSIZE_T_MAX on 64-bit machines, we - * refuse to allocate new blocks if the current len is dangerously - * close. There is some extra margin to prevent spurious arithmetic - * overflows at various places. The following check ensures that - * the blocks allocated to the deque, in the worst case, can only - * have PY_SSIZE_T_MAX-2 entries in total. - */ + /* To prevent len from overflowing PY_SSIZE_T_MAX on 32-bit machines, we + * refuse to allocate new blocks if the current len is nearing overflow. */ if (len >= PY_SSIZE_T_MAX - 2*BLOCKLEN) { PyErr_SetString(PyExc_OverflowError, "cannot add more blocks to the deque"); From b748297a08fba8a7d64e7d6659d48cb3b120c435 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 25 Jun 2013 11:35:44 -0700 Subject: [PATCH 192/480] reapply f1dc30a1be72 --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 8a43d9a028a..f175fcaa0ea 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -48,8 +48,8 @@ typedef struct BLOCK { struct BLOCK *leftlink; - struct BLOCK *rightlink; PyObject *data[BLOCKLEN]; + struct BLOCK *rightlink; } block; #define MAXFREEBLOCKS 10 From e54d46fd85f6c335d0e04a7714cc9d5865d867da Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 28 Jun 2013 18:59:19 -0400 Subject: [PATCH 193/480] Issue #18315: Improve fileinput docs by adding 'bufsize' where missing and replacing redundant signature in input() docstring with one-line summary. Original patch by Terrel Shumway. --- Doc/library/fileinput.rst | 4 ++-- Lib/fileinput.py | 11 +++++------ Misc/ACKS | 1 + 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 172a643ac50..ce892ce277c 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -50,7 +50,7 @@ provided by this module. The following function is the primary interface of this module: -.. function:: input([files[, inplace[, backup[, mode[, openhook]]]]]) +.. function:: input([files[, inplace[, backup[, bufsize[, mode[, openhook]]]]]]) Create an instance of the :class:`FileInput` class. The instance will be used as global state for the functions of this module, and is also returned to use @@ -122,7 +122,7 @@ The class which implements the sequence behavior provided by the module is available for subclassing as well: -.. class:: FileInput([files[, inplace[, backup[, mode[, openhook]]]]]) +.. class:: FileInput([files[, inplace[, backup[,bufsize[, mode[, openhook]]]]]]) Class :class:`FileInput` is the implementation; its methods :meth:`filename`, :meth:`fileno`, :meth:`lineno`, :meth:`filelineno`, :meth:`isfirstline`, diff --git a/Lib/fileinput.py b/Lib/fileinput.py index ba485752092..04e97bdb418 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -90,12 +90,11 @@ def input(files=None, inplace=0, backup="", bufsize=0, mode="r", openhook=None): - """input([files[, inplace[, backup[, mode[, openhook]]]]]) + """Return an instance of the FileInput class, which can be iterated. - Create an instance of the FileInput class. The instance will be used - as global state for the functions of this module, and is also returned - to use during iteration. The parameters to this function will be passed - along to the constructor of the FileInput class. + The parameters are passed to the constructor of the FileInput class. + The returned instance, in addition to being an iterator, + keeps global state for the functions of this module,. """ global _state if _state and _state._file: @@ -182,7 +181,7 @@ def isstdin(): return _state.isstdin() class FileInput: - """class FileInput([files[, inplace[, backup[, mode[, openhook]]]]]) + """FileInput([files[, inplace[, backup[, bufsize[, mode[, openhook]]]]]]) Class FileInput is the implementation of the module; its methods filename(), lineno(), fileline(), isfirstline(), isstdin(), fileno(), diff --git a/Misc/ACKS b/Misc/ACKS index fc0d9d216be..f8e588e1dd7 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -937,6 +937,7 @@ John W. Shipman Joel Shprentz Itamar Shtull-Trauring Yue Shuaijie +Terrel Shumway Eric Siegerman Paul Sijben Tim Silk From 2b160bcb39b2089422c86ed9b575621199ff0868 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 28 Jun 2013 23:51:34 -0400 Subject: [PATCH 194/480] Issue *18081, #18242: Change Idle warnings capture in PyShell and run to stop replacing warnings.formatwarnings and to reverse replacement of warnings.showwarnings when import is complete and when main function exits. Add test_warning.py. Vinay Sajip provided capture_warnings function. --- Lib/idlelib/PyShell.py | 82 +++++++++++++++++---------- Lib/idlelib/idle_test/test_warning.py | 73 ++++++++++++++++++++++++ Lib/idlelib/run.py | 56 ++++++++++++------ 3 files changed, 164 insertions(+), 47 deletions(-) create mode 100644 Lib/idlelib/idle_test/test_warning.py diff --git a/Lib/idlelib/PyShell.py b/Lib/idlelib/PyShell.py index 1ae69499bc4..e936bd25a2f 100644 --- a/Lib/idlelib/PyShell.py +++ b/Lib/idlelib/PyShell.py @@ -50,35 +50,55 @@ # internal warnings to the console. ScriptBinding.check_syntax() will # temporarily redirect the stream to the shell window to display warnings when # checking user's code. -global warning_stream -warning_stream = sys.__stderr__ -try: - import warnings -except ImportError: - pass -else: - def idle_showwarning(message, category, filename, lineno, - file=None, line=None): - if file is None: - file = warning_stream - try: - file.write(warnings.formatwarning(message, category, filename, - lineno, line=line)) - except IOError: - pass ## file (probably __stderr__) is invalid, warning dropped. - warnings.showwarning = idle_showwarning - def idle_formatwarning(message, category, filename, lineno, line=None): - """Format warnings the IDLE way""" - s = "\nWarning (from warnings module):\n" - s += ' File \"%s\", line %s\n' % (filename, lineno) - if line is None: - line = linecache.getline(filename, lineno) - line = line.strip() - if line: - s += " %s\n" % line - s += "%s: %s\n>>> " % (category.__name__, message) - return s - warnings.formatwarning = idle_formatwarning +warning_stream = sys.__stderr__ # None, at least on Windows, if no console. +import warnings + +def idle_formatwarning(message, category, filename, lineno, line=None): + """Format warnings the IDLE way.""" + + s = "\nWarning (from warnings module):\n" + s += ' File \"%s\", line %s\n' % (filename, lineno) + if line is None: + line = linecache.getline(filename, lineno) + line = line.strip() + if line: + s += " %s\n" % line + s += "%s: %s\n" % (category.__name__, message) + return s + +def idle_showwarning( + message, category, filename, lineno, file=None, line=None): + """Show Idle-format warning (after replacing warnings.showwarning). + + The differences are the formatter called, the file=None replacement, + which can be None, the capture of the consequence AttributeError, + and the output of a hard-coded prompt. + """ + if file is None: + file = warning_stream + try: + file.write(idle_formatwarning( + message, category, filename, lineno, line=line)) + file.write(">>> ") + except (AttributeError, IOError): + pass # if file (probably __stderr__) is invalid, skip warning. + +_warnings_showwarning = None + +def capture_warnings(capture): + "Replace warning.showwarning with idle_showwarning, or reverse." + + global _warnings_showwarning + if capture: + if _warnings_showwarning is None: + _warnings_showwarning = warnings.showwarning + warnings.showwarning = idle_showwarning + else: + if _warnings_showwarning is not None: + warnings.showwarning = _warnings_showwarning + _warnings_showwarning = None + +capture_warnings(True) def extended_linecache_checkcache(filename=None, orig_checkcache=linecache.checkcache): @@ -1428,6 +1448,7 @@ def close(self): def main(): global flist, root, use_subprocess + capture_warnings(True) use_subprocess = True enable_shell = False enable_edit = False @@ -1560,7 +1581,10 @@ def main(): while flist.inversedict: # keep IDLE running while files are open. root.mainloop() root.destroy() + capture_warnings(False) if __name__ == "__main__": sys.modules['PyShell'] = sys.modules['__main__'] main() + +capture_warnings(False) # Make sure turned off; see issue 18081 diff --git a/Lib/idlelib/idle_test/test_warning.py b/Lib/idlelib/idle_test/test_warning.py new file mode 100644 index 00000000000..da1d8a1d0ab --- /dev/null +++ b/Lib/idlelib/idle_test/test_warning.py @@ -0,0 +1,73 @@ +'''Test warnings replacement in PyShell.py and run.py. + +This file could be expanded to include traceback overrides +(in same two modules). If so, change name. +Revise if output destination changes (http://bugs.python.org/issue18318). +Make sure warnings module is left unaltered (http://bugs.python.org/issue18081). +''' + +import unittest +from test.test_support import captured_stderr + +import warnings +# Try to capture default showwarning before Idle modules are imported. +showwarning = warnings.showwarning +# But if we run this file within idle, we are in the middle of the run.main loop +# and default showwarnings has already been replaced. +running_in_idle = 'idle' in showwarning.__name__ + +from idlelib import run +from idlelib import PyShell as shell + +# The following was generated from PyShell.idle_formatwarning +# and checked as matching expectation. +idlemsg = ''' +Warning (from warnings module): + File "test_warning.py", line 99 + Line of code +UserWarning: Test +''' +shellmsg = idlemsg + ">>> " + +class RunWarnTest(unittest.TestCase): + + @unittest.skipIf(running_in_idle, "Does not work when run within Idle.") + def test_showwarnings(self): + self.assertIs(warnings.showwarning, showwarning) + run.capture_warnings(True) + self.assertIs(warnings.showwarning, run.idle_showwarning_subproc) + run.capture_warnings(False) + self.assertIs(warnings.showwarning, showwarning) + + def test_run_show(self): + with captured_stderr() as f: + run.idle_showwarning_subproc( + 'Test', UserWarning, 'test_warning.py', 99, f, 'Line of code') + # The following uses .splitlines to erase line-ending differences + self.assertEqual(idlemsg.splitlines(), f.getvalue().splitlines()) + +class ShellWarnTest(unittest.TestCase): + + @unittest.skipIf(running_in_idle, "Does not work when run within Idle.") + def test_showwarnings(self): + self.assertIs(warnings.showwarning, showwarning) + shell.capture_warnings(True) + self.assertIs(warnings.showwarning, shell.idle_showwarning) + shell.capture_warnings(False) + self.assertIs(warnings.showwarning, showwarning) + + def test_idle_formatter(self): + # Will fail if format changed without regenerating idlemsg + s = shell.idle_formatwarning( + 'Test', UserWarning, 'test_warning.py', 99, 'Line of code') + self.assertEqual(idlemsg, s) + + def test_shell_show(self): + with captured_stderr() as f: + shell.idle_showwarning( + 'Test', UserWarning, 'test_warning.py', 99, f, 'Line of code') + self.assertEqual(shellmsg.splitlines(), f.getvalue().splitlines()) + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 9cc009fb7b8..e3755011fb0 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -22,24 +22,38 @@ LOCALHOST = '127.0.0.1' -try: - import warnings -except ImportError: - pass -else: - def idle_formatwarning_subproc(message, category, filename, lineno, - line=None): - """Format warnings the IDLE way""" - s = "\nWarning (from warnings module):\n" - s += ' File \"%s\", line %s\n' % (filename, lineno) - if line is None: - line = linecache.getline(filename, lineno) - line = line.strip() - if line: - s += " %s\n" % line - s += "%s: %s\n" % (category.__name__, message) - return s - warnings.formatwarning = idle_formatwarning_subproc +import warnings + +def idle_showwarning_subproc( + message, category, filename, lineno, file=None, line=None): + """Show Idle-format warning after replacing warnings.showwarning. + + The only difference is the formatter called. + """ + if file is None: + file = sys.stderr + try: + file.write(PyShell.idle_formatwarning( + message, category, filename, lineno, line)) + except IOError: + pass # the file (probably stderr) is invalid - this warning gets lost. + +_warnings_showwarning = None + +def capture_warnings(capture): + "Replace warning.showwarning with idle_showwarning_subproc, or reverse." + + global _warnings_showwarning + if capture: + if _warnings_showwarning is None: + _warnings_showwarning = warnings.showwarning + warnings.showwarning = idle_showwarning_subproc + else: + if _warnings_showwarning is not None: + warnings.showwarning = _warnings_showwarning + _warnings_showwarning = None + +capture_warnings(True) # Thread shared globals: Establish a queue between a subthread (which handles # the socket) and the main thread (which runs user code), plus global @@ -78,6 +92,8 @@ def main(del_exitfunc=False): except: print>>sys.stderr, "IDLE Subprocess: no IP port passed in sys.argv." return + + capture_warnings(True) sys.argv[:] = [""] sockthread = threading.Thread(target=manage_socket, name='SockThread', @@ -104,6 +120,7 @@ def main(del_exitfunc=False): exit_now = True continue except SystemExit: + capture_warnings(False) raise except: type, value, tb = sys.exc_info() @@ -219,6 +236,7 @@ def exit(): del sys.exitfunc except AttributeError: pass + capture_warnings(False) sys.exit(0) class MyRPCServer(rpc.RPCServer): @@ -352,3 +370,5 @@ def stackviewer(self, flist_oid=None): sys.last_value = val item = StackViewer.StackTreeItem(flist, tb) return RemoteObjectBrowser.remote_object_tree_item(item) + +capture_warnings(False) # Make sure turned off; see issue 18081 From 1517f47f6b096fe0517c501da4a27895b4ef90bf Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 29 Jun 2013 00:59:34 -0400 Subject: [PATCH 195/480] Issue #18316: Update idlelib 2.7 except clauses to ease backports. --- Lib/idlelib/EditorWindow.py | 2 +- Lib/idlelib/GrepDialog.py | 2 +- Lib/idlelib/IOBinding.py | 8 ++++---- Lib/idlelib/PyShell.py | 8 ++++---- Lib/idlelib/ScriptBinding.py | 6 +++--- Lib/idlelib/SearchEngine.py | 2 +- Lib/idlelib/run.py | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Lib/idlelib/EditorWindow.py b/Lib/idlelib/EditorWindow.py index 7b9b583db9a..14c76de87ff 100644 --- a/Lib/idlelib/EditorWindow.py +++ b/Lib/idlelib/EditorWindow.py @@ -689,7 +689,7 @@ def open_module(self, event=None): # XXX Ought to insert current file's directory in front of path try: (f, file, (suffix, mode, type)) = _find_module(name) - except (NameError, ImportError), msg: + except (NameError, ImportError) as msg: tkMessageBox.showerror("Import error", str(msg), parent=self.text) return if type != imp.PY_SOURCE: diff --git a/Lib/idlelib/GrepDialog.py b/Lib/idlelib/GrepDialog.py index c8b703c3ac8..b5e35d7454e 100644 --- a/Lib/idlelib/GrepDialog.py +++ b/Lib/idlelib/GrepDialog.py @@ -98,7 +98,7 @@ def grep_it(self, prog, path): def findfiles(self, dir, base, rec): try: names = os.listdir(dir or os.curdir) - except os.error, msg: + except os.error as msg: print msg return [] list = [] diff --git a/Lib/idlelib/IOBinding.py b/Lib/idlelib/IOBinding.py index 8078c3f4b76..f0d4d227c3c 100644 --- a/Lib/idlelib/IOBinding.py +++ b/Lib/idlelib/IOBinding.py @@ -251,7 +251,7 @@ def loadfile(self, filename): f = open(filename,'rb') chars = f.read() f.close() - except IOError, msg: + except IOError as msg: tkMessageBox.showerror("I/O Error", str(msg), master=self.text) return False @@ -294,7 +294,7 @@ def decode(self, chars): # Next look for coding specification try: enc = coding_spec(chars) - except LookupError, name: + except LookupError as name: tkMessageBox.showerror( title="Error loading the file", message="The encoding '%s' is not known to this Python "\ @@ -388,7 +388,7 @@ def writefile(self, filename): f.flush() f.close() return True - except IOError, msg: + except IOError as msg: tkMessageBox.showerror("I/O Error", str(msg), master=self.text) return False @@ -408,7 +408,7 @@ def encode(self, chars): try: enc = coding_spec(chars) failed = None - except LookupError, msg: + except LookupError as msg: failed = msg enc = None if enc: diff --git a/Lib/idlelib/PyShell.py b/Lib/idlelib/PyShell.py index e936bd25a2f..0acfe4a95a4 100644 --- a/Lib/idlelib/PyShell.py +++ b/Lib/idlelib/PyShell.py @@ -430,7 +430,7 @@ def start_subprocess(self): try: self.rpcclt = MyRPCClient(addr) break - except socket.error, err: + except socket.error as err: pass else: self.display_port_binding_error() @@ -451,7 +451,7 @@ def start_subprocess(self): self.rpcclt.listening_sock.settimeout(10) try: self.rpcclt.accept() - except socket.timeout, err: + except socket.timeout as err: self.display_no_subprocess_error() return None self.rpcclt.register("console", self.tkconsole) @@ -486,7 +486,7 @@ def restart_subprocess(self, with_cwd=False): self.spawn_subprocess() try: self.rpcclt.accept() - except socket.timeout, err: + except socket.timeout as err: self.display_no_subprocess_error() return None self.transfer_path(with_cwd=with_cwd) @@ -1458,7 +1458,7 @@ def main(): startup = False try: opts, args = getopt.getopt(sys.argv[1:], "c:deihnr:st:") - except getopt.error, msg: + except getopt.error as msg: sys.stderr.write("Error: %s\n" % str(msg)) sys.stderr.write(usage_msg) sys.exit(2) diff --git a/Lib/idlelib/ScriptBinding.py b/Lib/idlelib/ScriptBinding.py index 01ac4746589..a1d5aaf7f74 100644 --- a/Lib/idlelib/ScriptBinding.py +++ b/Lib/idlelib/ScriptBinding.py @@ -70,13 +70,13 @@ def tabnanny(self, filename): f = open(filename, 'r') try: tabnanny.process_tokens(tokenize.generate_tokens(f.readline)) - except tokenize.TokenError, msg: + except tokenize.TokenError as msg: msgtxt, (lineno, start) = msg self.editwin.gotoline(lineno) self.errorbox("Tabnanny Tokenizing Error", "Token Error: %s" % msgtxt) return False - except tabnanny.NannyNag, nag: + except tabnanny.NannyNag as nag: # The error messages from tabnanny are too confusing... self.editwin.gotoline(nag.get_lineno()) self.errorbox("Tab/space error", indent_message) @@ -101,7 +101,7 @@ def checksyntax(self, filename): try: # If successful, return the compiled code return compile(source, filename, "exec") - except (SyntaxError, OverflowError, ValueError), err: + except (SyntaxError, OverflowError, ValueError) as err: try: msg, (errorfilename, lineno, offset, line) = err if not errorfilename: diff --git a/Lib/idlelib/SearchEngine.py b/Lib/idlelib/SearchEngine.py index cc40a00c50f..6745faff021 100644 --- a/Lib/idlelib/SearchEngine.py +++ b/Lib/idlelib/SearchEngine.py @@ -66,7 +66,7 @@ def getprog(self): flags = flags | re.IGNORECASE try: prog = re.compile(pat, flags) - except re.error, what: + except re.error as what: try: msg, col = what except: diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index e3755011fb0..604c5cd54e1 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -140,7 +140,7 @@ def manage_socket(address): try: server = MyRPCServer(address, MyHandler) break - except socket.error, err: + except socket.error as err: print>>sys.__stderr__,"IDLE Subprocess: socket error: "\ + err.args[1] + ", retrying...." else: From 00a0bd2a273277e35cc2edab12498375f47bdb62 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 29 Jun 2013 13:15:36 -0400 Subject: [PATCH 196/480] Issue #18237: Fix assertRaisesRegexp error caought by Jeff Tratner. --- Doc/library/unittest.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 46f429c40cf..1bd3819bc2d 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -958,7 +958,7 @@ Test cases a regular expression object or a string containing a regular expression suitable for use by :func:`re.search`. Examples:: - self.assertRaisesRegexp(ValueError, 'invalid literal for.*XYZ$', + self.assertRaisesRegexp(ValueError, "invalid literal for.*XYZ'$", int, 'XYZ') or:: From 131c2f7a4af8113579c3efa4b1e74227d27e6a3c Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 29 Jun 2013 18:22:02 -0400 Subject: [PATCH 197/480] Issue #18103: Update README.txt and test_idle to describe and run gui tests. --- Lib/idlelib/idle_test/README.txt | 91 ++++++++++++++++++++++---------- Lib/test/test_idle.py | 5 ++ 2 files changed, 69 insertions(+), 27 deletions(-) diff --git a/Lib/idlelib/idle_test/README.txt b/Lib/idlelib/idle_test/README.txt index 1919f2f52b1..b498e7ce1f8 100644 --- a/Lib/idlelib/idle_test/README.txt +++ b/Lib/idlelib/idle_test/README.txt @@ -1,14 +1,19 @@ README FOR IDLE TESTS IN IDLELIB.IDLE_TEST + +1. Test Files + The idle directory, idlelib, has over 60 xyz.py files. The idle_test -subdirectory should contain a test_xyy.py for each one. (For test modules, -make 'xyz' lower case.) Each should start with the following cut-paste -template, with the blanks after after '.'. 'as', and '_' filled in. +subdirectory should contain a test_xyy.py for each. (For test modules, make +'xyz' lower case, and possibly shorten it.) Each file should start with the +something like the following template, with the blanks after after '.' and 'as', +and before and after '_' filled in. --- import unittest +from test.support import requires import idlelib. as -class Test_(unittest.TestCase): +class _Test(unittest.TestCase): def test_(self): @@ -21,43 +26,75 @@ Once test_xyy is written, the following should go at the end of xyy.py, with xyz (lowercased) added after 'test_'. --- if __name__ == "__main__": + from test import support; support.use_resources = ['gui'] import unittest unittest.main('idlelib.idle_test.test_', verbosity=2, exit=False) --- -In Idle, pressing F5 in an editor window with either xyz.py or test_xyz.py -loaded will then run the test with the version of Python running Idle and -tracebacks will appear in the Shell window. The options are appropriate for -developers running (as opposed to importing) either type of file during -development: verbosity=2 lists all test_y methods; exit=False avoids a -spurious sys.exit traceback when running in Idle. The following command -lines also run test_xyz.py + +2. Gui Tests + +Gui tests need 'requires' and 'use_resources' from test.support +(test.test_support in 2.7). A test is a gui test if it creates a Tk root or +master object either directly or indirectly by instantiating a tkinter or +idle class. For the benefit of buildbot machines that do not have a graphics +screen, gui tests must be 'guarded' by "requires('gui')" in a setUp +function or method. This will typically be setUpClass. +--- + @classmethod + def setUpClass(cls): + requires('gui') +--- +All gui objects must be destroyed by the end of the test, perhaps in a tearDown +function. + +Support.requires('gui') returns true if it is either called in a main module +(which never happens on buildbots) or if use_resources contains 'gui'. +Use_resources is set by test.regrtest but not by unittest. So when running +tests in another module with unittest, we set it ourselves, as in the xyz.py +template above. + +Since non-gui tests always run, but gui tests only sometimes, tests of non-gui +operations should best avoid needing a gui. Methods that make incidental use of +tkinter variables and messageboxes can do this by using the mock classes in +idle_test/mock_tk.py. + + +3. Running Tests + +Assume that xyz.py and test_xyz.py end with the "if __name__" statements given +above. In Idle, pressing F5 in an editor window with either loaded will run all +tests in the test_xyz file with the version of Python running Idle. The test +report and any tracebacks will appear in the Shell window. The options in these +"if __name__" statements are appropriate for developers running (as opposed to +importing) either of the files during development: verbosity=2 lists all test +methods in the file; exit=False avoids a spurious sys.exit traceback that would +otherwise occur when running in Idle. The following command lines also run +all test methods, including gui tests, in test_xyz.py. (The exceptions are that +idlelib and idlelib.idle start Idle and idlelib.PyShell should (issue 18330).) python -m idlelib.xyz # With the capitalization of the xyz module -python -m unittest -v idlelib.idle_test.test_xyz +python -m idlelib.idle_test.test_xyz -To run all idle tests either interactively ('>>>', with unittest imported) -or from a command line, use one of the following. +To run all idle_test/test_*.py tests, either interactively +('>>>', with unittest imported) or from a command line, use one of the +following. (Notes: unittest does not run gui tests; in 2.7, 'test ' (with the +space) is 'test.regrtest '; where present, -v and -ugui can be omitted.) >>> unittest.main('idlelib.idle_test', verbosity=2, exit=False) python -m unittest -v idlelib.idle_test +python -m test -v -ugui test_idle python -m test.test_idle -python -m test test_idle -The idle tests are 'discovered' in idlelib.idle_test.__init__.load_tests, +The idle tests are 'discovered' by idlelib.idle_test.__init__.load_tests, which is also imported into test.test_idle. Normally, neither file should be -changed when working on individual test modules. The last command runs runs +changed when working on individual test modules. The third command runs runs unittest indirectly through regrtest. The same happens when the entire test -suite is run with 'python -m test'. So it must work for buildbots to stay green. +suite is run with 'python -m test'. So that command must work for buildbots +to stay green. Idle tests must not disturb the environment in a way that +makes other tests fail (issue 18081). -To run an individual Testcase or test method, extend the -dotted name given to unittest on the command line. +To run an individual Testcase or test method, extend the dotted name given to +unittest on the command line. (But gui tests will not this way.) python -m unittest -v idlelib.idle_test.text_xyz.Test_case.test_meth - -To disable test/test_idle.py, there are at least two choices. -a. Comment out 'load_tests' line, no no tests are discovered (simple and safe); -Running no tests passes, so there is no indication that nothing was run. -b.Before that line, make module an unexpected skip for regrtest with -import unittest; raise unittest.SkipTest('skip for buildbots') -When run directly with unittest, this causes a normal exit and traceback. \ No newline at end of file diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 9eeb4e2e180..4262aa3f679 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -10,5 +10,10 @@ load_tests = idletest.load_tests if __name__ == '__main__': + # Until unittest supports resources, we emulate regrtest's -ugui + # so loaded tests run the same as if textually present here. + # If any Idle test ever needs another resource, add it to the list. + from test import support + support.use_resources = ['gui'] # use_resources is initially None import unittest unittest.main(verbosity=2, exit=False) From 1d6e98e6731268e06c406db2758a1605358cb0a4 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 29 Jun 2013 18:43:59 -0400 Subject: [PATCH 198/480] #18155: Regex-escape delimiter, in case it is a regex special char. Patch by Vajrasky Kok, with slight modification to the tests by me. --- Lib/csv.py | 5 +++-- Lib/test/test_csv.py | 42 +++++++++++++++++++++++++++++++++++++++--- Misc/ACKS | 1 + Misc/NEWS | 6 +++++- 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/Lib/csv.py b/Lib/csv.py index 984ed7e581b..98480ba16a1 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -261,8 +261,9 @@ def _guess_quote_and_delimiter(self, data, delimiters): # if we see an extra quote between delimiters, we've got a # double quoted format - dq_regexp = re.compile(r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ - {'delim':delim, 'quote':quotechar}, re.MULTILINE) + dq_regexp = re.compile( + r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ + {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 53ca5ab12f1..3f826651292 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -914,7 +914,7 @@ class TestSniffer(unittest.TestCase): 'Tommy''s Place':'Blue Island':'IL':'12/28/02':'Blue Sunday/White Crow' 'Stonecutters ''Seafood'' and Chop House':'Lemont':'IL':'12/19/02':'Week Back' """ - header = '''\ + header1 = '''\ "venue","city","state","date","performers" ''' sample3 = '''\ @@ -933,10 +933,35 @@ class TestSniffer(unittest.TestCase): sample6 = "a|b|c\r\nd|e|f\r\n" sample7 = "'a'|'b'|'c'\r\n'd'|e|f\r\n" +# Issue 18155: Use a delimiter that is a special char to regex: + + header2 = '''\ +"venue"+"city"+"state"+"date"+"performers" +''' + sample8 = """\ +Harry's+ Arlington Heights+ IL+ 2/1/03+ Kimi Hayes +Shark City+ Glendale Heights+ IL+ 12/28/02+ Prezence +Tommy's Place+ Blue Island+ IL+ 12/28/02+ Blue Sunday/White Crow +Stonecutters Seafood and Chop House+ Lemont+ IL+ 12/19/02+ Week Back +""" + sample9 = """\ +'Harry''s'+ Arlington Heights'+ 'IL'+ '2/1/03'+ 'Kimi Hayes' +'Shark City'+ Glendale Heights'+' IL'+ '12/28/02'+ 'Prezence' +'Tommy''s Place'+ Blue Island'+ 'IL'+ '12/28/02'+ 'Blue Sunday/White Crow' +'Stonecutters ''Seafood'' and Chop House'+ 'Lemont'+ 'IL'+ '12/19/02'+ 'Week Back' +""" + def test_has_header(self): sniffer = csv.Sniffer() self.assertEqual(sniffer.has_header(self.sample1), False) - self.assertEqual(sniffer.has_header(self.header+self.sample1), True) + self.assertEqual(sniffer.has_header(self.header1 + self.sample1), + True) + + def test_has_header_regex_special_delimiter(self): + sniffer = csv.Sniffer() + self.assertEqual(sniffer.has_header(self.sample8), False) + self.assertEqual(sniffer.has_header(self.header2 + self.sample8), + True) def test_sniff(self): sniffer = csv.Sniffer() @@ -970,13 +995,24 @@ def test_delimiters(self): dialect = sniffer.sniff(self.sample7) self.assertEqual(dialect.delimiter, "|") self.assertEqual(dialect.quotechar, "'") + dialect = sniffer.sniff(self.sample8) + self.assertEqual(dialect.delimiter, '+') + dialect = sniffer.sniff(self.sample9) + self.assertEqual(dialect.delimiter, '+') + self.assertEqual(dialect.quotechar, "'") def test_doublequote(self): sniffer = csv.Sniffer() - dialect = sniffer.sniff(self.header) + dialect = sniffer.sniff(self.header1) + self.assertFalse(dialect.doublequote) + dialect = sniffer.sniff(self.header2) self.assertFalse(dialect.doublequote) dialect = sniffer.sniff(self.sample2) self.assertTrue(dialect.doublequote) + dialect = sniffer.sniff(self.sample8) + self.assertFalse(dialect.doublequote) + dialect = sniffer.sniff(self.sample9) + self.assertTrue(dialect.doublequote) if not hasattr(sys, "gettotalrefcount"): if test_support.verbose: print "*** skipping leakage tests ***" diff --git a/Misc/ACKS b/Misc/ACKS index f8e588e1dd7..67f0c97eafa 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -545,6 +545,7 @@ Jeff Knupp Greg Kochanski Damon Kohler Marko Kohtala +Vajrasky Kok Guido Kollerie Peter A. Koren Joseph Koshy diff --git a/Misc/NEWS b/Misc/NEWS index fff0032c268..175fa9569b2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,11 +24,15 @@ Core and Builtins Library ------- +- Issue #18155: The csv module now correctly handles csv files that use + a delimiter character that has a special meaning in regexes, instead of + throwing an exception. + - Issue #18135: ssl.SSLSocket.write() now raises an OverflowError if the input string in longer than 2 gigabytes. The ssl module does not support partial write. -- Issue #18167: cgi.FieldStorage no more fails to handle multipart/form-data +- Issue #18167: cgi.FieldStorage no longer fails to handle multipart/form-data when \r\n appears at end of 65535 bytes without other newlines. - Issue #17403: urllib.parse.robotparser normalizes the urls before adding to From ab75d5de106c223638a4d50f05fbbda78ef03fe5 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 30 Jun 2013 13:57:57 -0400 Subject: [PATCH 199/480] Tempfile.py: stop buildbot warning about using deprecated xreadlines. The slightly odd behavior (the validity of passing a sizehint depends on the type of self._file) was kept to avoid breaking code that depends on it. Test_tempfile.test_xreadlines passes (along with everything else). --- Lib/tempfile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/tempfile.py b/Lib/tempfile.py index f2ddbb0d9ca..b5dce974e18 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -612,7 +612,7 @@ def writelines(self, iterable): return rv def xreadlines(self, *args): - try: - return self._file.xreadlines(*args) - except AttributeError: + if hasattr(self._file, 'xreadlines'): # real file + return iter(self._file) + else: # StringIO() return iter(self._file.readlines(*args)) From 482ea21e170882e1c7be35ced0763a5afa79f866 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 30 Jun 2013 16:51:53 -0400 Subject: [PATCH 200/480] Issue 18189: remove unused methods in idlelib.Delegator.Delegator. Idle code already uses x.delegate instead of x.getdelegate(). The printed report must have been for testing. --- Lib/idlelib/Delegator.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Lib/idlelib/Delegator.py b/Lib/idlelib/Delegator.py index 6125591fe0d..60a2cad2254 100644 --- a/Lib/idlelib/Delegator.py +++ b/Lib/idlelib/Delegator.py @@ -20,14 +20,6 @@ def resetcache(self): pass self.__cache.clear() - def cachereport(self): - keys = self.__cache.keys() - keys.sort() - print keys - def setdelegate(self, delegate): self.resetcache() self.delegate = delegate - - def getdelegate(self): - return self.delegate From 3f2f2bd111963f4d64b8ff6b931292797fbcc81c Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 30 Jun 2013 18:36:53 -0400 Subject: [PATCH 201/480] Issue #18189: add test_delegator for Idle Delegator class. Also change private dict used as a set to a set. --- Lib/idlelib/Delegator.py | 6 ++-- Lib/idlelib/idle_test/test_delegator.py | 37 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 Lib/idlelib/idle_test/test_delegator.py diff --git a/Lib/idlelib/Delegator.py b/Lib/idlelib/Delegator.py index 60a2cad2254..c4765163f80 100644 --- a/Lib/idlelib/Delegator.py +++ b/Lib/idlelib/Delegator.py @@ -4,16 +4,16 @@ class Delegator: def __init__(self, delegate=None): self.delegate = delegate - self.__cache = {} + self.__cache = set() def __getattr__(self, name): attr = getattr(self.delegate, name) # May raise AttributeError setattr(self, name, attr) - self.__cache[name] = attr + self.__cache.add(name) return attr def resetcache(self): - for key in self.__cache.keys(): + for key in self.__cache: try: delattr(self, key) except AttributeError: diff --git a/Lib/idlelib/idle_test/test_delegator.py b/Lib/idlelib/idle_test/test_delegator.py new file mode 100644 index 00000000000..b8ae5eeefe3 --- /dev/null +++ b/Lib/idlelib/idle_test/test_delegator.py @@ -0,0 +1,37 @@ +import unittest +from idlelib.Delegator import Delegator + +class DelegatorTest(unittest.TestCase): + + def test_mydel(self): + # test a simple use scenario + + # initialize + mydel = Delegator(int) + self.assertIs(mydel.delegate, int) + self.assertEqual(mydel._Delegator__cache, set()) + + # add an attribute: + self.assertRaises(AttributeError, mydel.__getattr__, 'xyz') + bl = mydel.bit_length + self.assertIs(bl, int.bit_length) + self.assertIs(mydel.__dict__['bit_length'], int.bit_length) + self.assertEqual(mydel._Delegator__cache, {'bit_length'}) + + # add a second attribute + mydel.numerator + self.assertEqual(mydel._Delegator__cache, {'bit_length', 'numerator'}) + + # delete the second (which, however, leaves it in the name cache) + del mydel.numerator + self.assertNotIn('numerator', mydel.__dict__) + self.assertIn('numerator', mydel._Delegator__cache) + + # reset by calling .setdelegate, which calls .resetcache + mydel.setdelegate(float) + self.assertIs(mydel.delegate, float) + self.assertNotIn('bit_length', mydel.__dict__) + self.assertEqual(mydel._Delegator__cache, set()) + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) From e2772533df0677da78659a233a252be095ce73ef Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 30 Jun 2013 19:07:49 -0400 Subject: [PATCH 202/480] Issue #8515: Set __file__ when run file in IDLE. Backport 2c276d0553ff by Andrew Svetlov, based on initial patch by Bruce Frederiksen. --- Lib/idlelib/ScriptBinding.py | 12 ++++++------ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Lib/idlelib/ScriptBinding.py b/Lib/idlelib/ScriptBinding.py index a1d5aaf7f74..665b3b241cc 100644 --- a/Lib/idlelib/ScriptBinding.py +++ b/Lib/idlelib/ScriptBinding.py @@ -152,16 +152,16 @@ def run_module_event(self, event): dirname = os.path.dirname(filename) # XXX Too often this discards arguments the user just set... interp.runcommand("""if 1: - _filename = %r + __file__ = {filename!r} import sys as _sys from os.path import basename as _basename if (not _sys.argv or - _basename(_sys.argv[0]) != _basename(_filename)): - _sys.argv = [_filename] + _basename(_sys.argv[0]) != _basename(__file__)): + _sys.argv = [__file__] import os as _os - _os.chdir(%r) - del _filename, _sys, _basename, _os - \n""" % (filename, dirname)) + _os.chdir({dirname!r}) + del _sys, _basename, _os + \n""".format(filename=filename, dirname=dirname)) interp.prepend_syspath(filename) # XXX KBK 03Jul04 When run w/o subprocess, runtime warnings still # go to __stderr__. With subprocess, they go to the shell. diff --git a/Misc/ACKS b/Misc/ACKS index 67f0c97eafa..82e49566f07 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -317,6 +317,7 @@ Doug Fort John Fouhy Stefan Franke Martin Franklin +Bruce Frederiksen Robin Friedrich Bradley Froehle Ivan Frohne diff --git a/Misc/NEWS b/Misc/NEWS index 175fa9569b2..9048a9862a2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -73,6 +73,9 @@ Build IDLE ---- +- Issue #8515: Set __file__ when run file in IDLE. + Initial patch by Bruce Frederiksen. + - Issue #5492: Avoid traceback when exiting IDLE caused by a race condition. - Issue #17511: Keep IDLE find dialog open after clicking "Find Next". From 8e6219f68970b3dfdc1edd118b4d82ef86690513 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 1 Jul 2013 00:42:44 -0400 Subject: [PATCH 203/480] Issue #7136: In the Idle File menu, "New Window" is renamed "New File". Patch by Tal Einat, Roget Serwy, and Todd Rovito. --- Doc/library/idle.rst | 4 ++-- Lib/idlelib/Bindings.py | 2 +- Lib/idlelib/help.txt | 2 +- Misc/NEWS | 3 +++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index b40e4701536..36d78b0991d 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -33,8 +33,8 @@ Menus File menu ^^^^^^^^^ -New window - create a new editing window +New file + create a new file editing window Open... open an existing file diff --git a/Lib/idlelib/Bindings.py b/Lib/idlelib/Bindings.py index fe6e812d2bf..65c0317e60c 100644 --- a/Lib/idlelib/Bindings.py +++ b/Lib/idlelib/Bindings.py @@ -15,7 +15,7 @@ menudefs = [ # underscore prefixes character to underscore ('file', [ - ('_New Window', '<>'), + ('_New File', '<>'), ('_Open...', '<>'), ('Open _Module...', '<>'), ('Class _Browser', '<>'), diff --git a/Lib/idlelib/help.txt b/Lib/idlelib/help.txt index 4b42e0536cc..bd6822c4233 100644 --- a/Lib/idlelib/help.txt +++ b/Lib/idlelib/help.txt @@ -5,7 +5,7 @@ separate window containing the menu is created. File Menu: - New Window -- Create a new editing window + New File -- Create a new editing window Open... -- Open an existing file Recent Files... -- Open a list of recent files Open Module... -- Open an existing module (searches sys.path) diff --git a/Misc/NEWS b/Misc/NEWS index 9048a9862a2..303760c07e6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -73,6 +73,9 @@ Build IDLE ---- +- Issue #7136: In the Idle File menu, "New Window" is renamed "New File". + Patch by Tal Einat, Roget Serwy, and Todd Rovito. + - Issue #8515: Set __file__ when run file in IDLE. Initial patch by Bruce Frederiksen. From 1d92962910f766749a89aed5a1a5bac283368c53 Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Mon, 1 Jul 2013 18:45:28 +0100 Subject: [PATCH 204/480] Issue #17097: Make multiprocessing ignore EINTR. --- Lib/multiprocessing/connection.py | 9 ++- Lib/test/test_multiprocessing.py | 70 +++++++++++++++++++- Misc/NEWS | 2 + Modules/_multiprocessing/socket_connection.c | 57 +++++++++++++--- 4 files changed, 127 insertions(+), 11 deletions(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 57bf811eda5..a0517301510 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -270,7 +270,14 @@ def __init__(self, address, family, backlog=1): self._unlink = None def accept(self): - s, self._last_accepted = self._socket.accept() + while True: + try: + s, self._last_accepted = self._socket.accept() + except socket.error as e: + if e.args[0] != errno.EINTR: + raise + else: + break s.setblocking(True) fd = duplicate(s.fileno()) conn = _multiprocessing.Connection(fd) diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index 88663ef55bb..5ffe75994c4 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -2460,13 +2460,81 @@ def test_lock(self): p.join() self.assertLessEqual(new_size, old_size) +# +# Issue #17097: EINTR should be ignored by recv(), send(), accept() etc +# + +class TestIgnoreEINTR(unittest.TestCase): + + @classmethod + def _test_ignore(cls, conn): + def handler(signum, frame): + pass + signal.signal(signal.SIGUSR1, handler) + conn.send('ready') + x = conn.recv() + conn.send(x) + conn.send_bytes(b'x'*(1024*1024)) # sending 1 MB should block + + @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') + def test_ignore(self): + conn, child_conn = multiprocessing.Pipe() + try: + p = multiprocessing.Process(target=self._test_ignore, + args=(child_conn,)) + p.daemon = True + p.start() + child_conn.close() + self.assertEqual(conn.recv(), 'ready') + time.sleep(0.1) + os.kill(p.pid, signal.SIGUSR1) + time.sleep(0.1) + conn.send(1234) + self.assertEqual(conn.recv(), 1234) + time.sleep(0.1) + os.kill(p.pid, signal.SIGUSR1) + self.assertEqual(conn.recv_bytes(), b'x'*(1024*1024)) + time.sleep(0.1) + p.join() + finally: + conn.close() + + @classmethod + def _test_ignore_listener(cls, conn): + def handler(signum, frame): + pass + signal.signal(signal.SIGUSR1, handler) + l = multiprocessing.connection.Listener() + conn.send(l.address) + a = l.accept() + a.send('welcome') + + @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') + def test_ignore_listener(self): + conn, child_conn = multiprocessing.Pipe() + try: + p = multiprocessing.Process(target=self._test_ignore_listener, + args=(child_conn,)) + p.daemon = True + p.start() + child_conn.close() + address = conn.recv() + time.sleep(0.1) + os.kill(p.pid, signal.SIGUSR1) + time.sleep(0.1) + client = multiprocessing.connection.Client(address) + self.assertEqual(client.recv(), 'welcome') + p.join() + finally: + conn.close() + # # # testcases_other = [OtherTest, TestInvalidHandle, TestInitializers, TestStdinBadfiledescriptor, TestTimeouts, TestNoForkBomb, - TestFlags, TestForkAwareThreadLock] + TestFlags, TestForkAwareThreadLock, TestIgnoreEINTR] # # diff --git a/Misc/NEWS b/Misc/NEWS index 303760c07e6..5c9e3677753 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,6 +24,8 @@ Core and Builtins Library ------- +- Issue #17097: Make multiprocessing ignore EINTR. + - Issue #18155: The csv module now correctly handles csv files that use a delimiter character that has a special meaning in regexes, instead of throwing an exception. diff --git a/Modules/_multiprocessing/socket_connection.c b/Modules/_multiprocessing/socket_connection.c index 66bc3777924..bdb0a32e375 100644 --- a/Modules/_multiprocessing/socket_connection.c +++ b/Modules/_multiprocessing/socket_connection.c @@ -22,6 +22,21 @@ # define CLOSE(h) close(h) #endif +/* + * Wrapper for PyErr_CheckSignals() which can be called without the GIL + */ + +static int +check_signals(void) +{ + PyGILState_STATE state; + int res; + state = PyGILState_Ensure(); + res = PyErr_CheckSignals(); + PyGILState_Release(state); + return res; +} + /* * Send string to file descriptor */ @@ -34,8 +49,14 @@ _conn_sendall(HANDLE h, char *string, size_t length) while (length > 0) { res = WRITE(h, p, length); - if (res < 0) + if (res < 0) { + if (errno == EINTR) { + if (check_signals() < 0) + return MP_EXCEPTION_HAS_BEEN_SET; + continue; + } return MP_SOCKET_ERROR; + } length -= res; p += res; } @@ -56,12 +77,16 @@ _conn_recvall(HANDLE h, char *buffer, size_t length) while (remaining > 0) { temp = READ(h, p, remaining); - if (temp <= 0) { - if (temp == 0) - return remaining == length ? - MP_END_OF_FILE : MP_EARLY_END_OF_FILE; - else - return temp; + if (temp < 0) { + if (errno == EINTR) { + if (check_signals() < 0) + return MP_EXCEPTION_HAS_BEEN_SET; + continue; + } + return temp; + } + else if (temp == 0) { + return remaining == length ? MP_END_OF_FILE : MP_EARLY_END_OF_FILE; } remaining -= temp; p += temp; @@ -171,9 +196,16 @@ conn_poll(ConnectionObject *conn, double timeout, PyThreadState *_save) p.revents = 0; if (timeout < 0) { - res = poll(&p, 1, -1); + do { + res = poll(&p, 1, -1); + } while (res < 0 && errno == EINTR); } else { res = poll(&p, 1, (int)(timeout * 1000 + 0.5)); + if (res < 0 && errno == EINTR) { + /* We were interrupted by a signal. Just indicate a + timeout even though we are early. */ + return FALSE; + } } if (res < 0) { @@ -209,12 +241,19 @@ conn_poll(ConnectionObject *conn, double timeout, PyThreadState *_save) FD_SET((SOCKET)conn->handle, &rfds); if (timeout < 0.0) { - res = select((int)conn->handle+1, &rfds, NULL, NULL, NULL); + do { + res = select((int)conn->handle+1, &rfds, NULL, NULL, NULL); + } while (res < 0 && errno == EINTR); } else { struct timeval tv; tv.tv_sec = (long)timeout; tv.tv_usec = (long)((timeout - tv.tv_sec) * 1e6 + 0.5); res = select((int)conn->handle+1, &rfds, NULL, NULL, &tv); + if (res < 0 && errno == EINTR) { + /* We were interrupted by a signal. Just indicate a + timeout even though we are early. */ + return FALSE; + } } if (res < 0) { From 756e8d14758f72f43b31c44490e7945bb4bd9a88 Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Tue, 2 Jul 2013 12:31:50 +0100 Subject: [PATCH 205/480] Issue #17273: Clarify that pool methods can only be used by parent process. --- Doc/library/multiprocessing.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 3f368a316a4..0756470c94a 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -287,6 +287,9 @@ For example:: print result.get(timeout=1) # prints "100" unless your computer is *very* slow print pool.map(f, range(10)) # prints "[0, 1, 4,..., 81]" +Note that the methods of a pool should only ever be used by the +process which created it. + Reference --------- @@ -1599,6 +1602,9 @@ with the :class:`Pool` class. *initializer* is not ``None`` then each worker process will call ``initializer(*initargs)`` when it starts. + Note that the methods of the pool object should only be called by + the process which created the pool. + .. versionadded:: 2.7 *maxtasksperchild* is the number of tasks a worker process can complete before it will exit and be replaced with a fresh worker process, to enable From d1a26d7690ab1b61143163f100237c39d6d25722 Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Tue, 2 Jul 2013 12:58:21 +0100 Subject: [PATCH 206/480] Issue #14206: Clarify docs for Queue.join_cancel_thread(). --- Doc/library/multiprocessing.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 0756470c94a..6e8c3a586e7 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -630,6 +630,13 @@ For an example of the usage of queues for interprocess communication see the background thread from being joined automatically when the process exits -- see :meth:`join_thread`. + A better name for this method might be + ``allow_exit_without_flush()``. It is likely to cause enqueued + data to lost, and you almost certainly will not need to use it. + It is really only there if you need the current process to exit + immediately without waiting to flush enqueued data to the + underlying pipe, and you don't care about lost data. + .. class:: multiprocessing.queues.SimpleQueue() From be7019b8050653d4ba8450b6c04487fe4d8654fd Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Tue, 2 Jul 2013 13:31:43 +0100 Subject: [PATCH 207/480] Issue #17261: Ensure multiprocessing's proxies use proper address. --- Lib/multiprocessing/managers.py | 1 + Misc/NEWS | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index ffe5812b89d..08d35d86828 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -763,6 +763,7 @@ def _callmethod(self, methodname, args=(), kwds={}): elif kind == '#PROXY': exposed, token = result proxytype = self._manager._registry[token.typeid][-1] + token.address = self._token.address proxy = proxytype( token, self._serializer, manager=self._manager, authkey=self._authkey, exposed=exposed diff --git a/Misc/NEWS b/Misc/NEWS index 5c9e3677753..405bf5c493e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,6 +24,8 @@ Core and Builtins Library ------- +- Issue #17261: Ensure multiprocessing's proxies use proper address. + - Issue #17097: Make multiprocessing ignore EINTR. - Issue #18155: The csv module now correctly handles csv files that use From edcf4f5edebf53e3697c8684c4d026fb0abd1263 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Tue, 2 Jul 2013 09:02:03 -0400 Subject: [PATCH 208/480] Closes #18312: 'make distclean' no longer deletes files in dot-directories. --- Makefile.pre.in | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 9d555503c14..98711ae62bb 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1295,11 +1295,11 @@ distclean: clobber Modules/ld_so_aix Modules/python.exp Misc/python.pc -rm -f python*-gdb.py -rm -f pybuilddir.txt - find $(srcdir) '(' -name '*.fdc' -o -name '*~' \ - -o -name '[@,#]*' -o -name '*.old' \ - -o -name '*.orig' -o -name '*.rej' \ - -o -name '*.bak' ')' \ - -exec rm -f {} ';' + find $(srcdir)/[a-zA-Z]* '(' -name '*.fdc' -o -name '*~' \ + -o -name '[@,#]*' -o -name '*.old' \ + -o -name '*.orig' -o -name '*.rej' \ + -o -name '*.bak' ')' \ + -exec rm -f {} ';' # Check for smelly exported symbols (not starting with Py/_Py) smelly: all From bb56e459d4a2d6e4c75285960cc25bdff6b8b847 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 4 Jul 2013 21:03:10 +0200 Subject: [PATCH 209/480] =?UTF-8?q?Issue=20#11185:=20Fix=20test=5Fwait4=20?= =?UTF-8?q?under=20AIX.=20=20Patch=20by=20S=C3=A9bastien=20Sabl=C3=A9.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/test/test_wait4.py | 8 +++++++- Misc/NEWS | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_wait4.py b/Lib/test/test_wait4.py index d04a11bde10..54580a90266 100644 --- a/Lib/test/test_wait4.py +++ b/Lib/test/test_wait4.py @@ -3,6 +3,7 @@ import os import time +import sys from test.fork_wait import ForkWait from test.test_support import run_unittest, reap_children, get_attribute @@ -13,10 +14,15 @@ class Wait4Test(ForkWait): def wait_impl(self, cpid): + option = os.WNOHANG + if sys.platform.startswith('aix'): + # Issue #11185: wait4 is broken on AIX and will always return 0 + # with WNOHANG. + option = 0 for i in range(10): # wait4() shouldn't hang, but some of the buildbots seem to hang # in the forking tests. This is an attempt to fix the problem. - spid, status, rusage = os.wait4(cpid, os.WNOHANG) + spid, status, rusage = os.wait4(cpid, option) if spid == cpid: break time.sleep(1.0) diff --git a/Misc/NEWS b/Misc/NEWS index 405bf5c493e..fec41b73424 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -99,6 +99,8 @@ IDLE Tests ----- +- Issue #11185: Fix test_wait4 under AIX. Patch by Sébastien Sablé. + - Issue #18094: test_uuid no more reports skipped tests as passed. - Issue #11995: test_pydoc doesn't import all sys.path modules anymore. From 180db5bd74c4244b72a538b6b7064b10dbd453dd Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 5 Jul 2013 01:41:30 +0200 Subject: [PATCH 210/480] Issue #18347: ElementTree's html serializer now preserves the case of closing tags. --- Lib/test/test_xml_etree.py | 10 ++++++++++ Lib/xml/etree/ElementTree.py | 6 +++--- Misc/NEWS | 3 +++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 4e161ca2f3a..4f06d20c045 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1769,6 +1769,16 @@ def bug_200709_iter_comment(): """ +def bug_18347(): + """ + + >>> e = ET.XML('text') + >>> serialize(e) + 'text' + >>> serialize(e, method="html") + 'text' + """ + # -------------------------------------------------------------------- # reported on bugs.python.org diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index bb468cde698..9f3e75d4a6f 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -988,15 +988,15 @@ def _serialize_html(write, elem, encoding, qnames, namespaces): # FIXME: handle boolean attributes write(" %s=\"%s\"" % (qnames[k], v)) write(">") - tag = tag.lower() + ltag = tag.lower() if text: - if tag == "script" or tag == "style": + if ltag == "script" or ltag == "style": write(_encode(text, encoding)) else: write(_escape_cdata(text, encoding)) for e in elem: _serialize_html(write, e, encoding, qnames, None) - if tag not in HTML_EMPTY: + if ltag not in HTML_EMPTY: write("") if elem.tail: write(_escape_cdata(elem.tail, encoding)) diff --git a/Misc/NEWS b/Misc/NEWS index fec41b73424..8114b696af8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,6 +24,9 @@ Core and Builtins Library ------- +- Issue #18347: ElementTree's html serializer now preserves the case of + closing tags. + - Issue #17261: Ensure multiprocessing's proxies use proper address. - Issue #17097: Make multiprocessing ignore EINTR. From 186bf6decfbd9b7541cd26b8454e1606bb9bbbc0 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sat, 6 Jul 2013 13:19:58 +0200 Subject: [PATCH 211/480] Issue #12990: The "Python Launcher" on OSX could not launch python scripts that have paths that include wide characters. --- Mac/PythonLauncher/MyDocument.m | 2 +- Misc/NEWS | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Mac/PythonLauncher/MyDocument.m b/Mac/PythonLauncher/MyDocument.m index 86112c4b15d..8f851e74e30 100755 --- a/Mac/PythonLauncher/MyDocument.m +++ b/Mac/PythonLauncher/MyDocument.m @@ -76,7 +76,7 @@ - (BOOL)run const char *cmdline; int sts; - cmdline = [[settings commandLineForScript: script] cString]; + cmdline = [[settings commandLineForScript: script] UTF8String]; if ([settings with_terminal]) { sts = doscript(cmdline); } else { diff --git a/Misc/NEWS b/Misc/NEWS index 8114b696af8..c3689e93188 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -67,6 +67,12 @@ Library - Issue #18113: Fixed a refcount leak in the curses.panel module's set_userptr() method. Reported by Atsuo Ishimoto. +Tools/Demos +----------- + +- Issue #12990: The "Python Launcher" on OSX could not launch python scripts + that have paths that include wide characters. + Build ----- From 1df044071cb0da5c3fb09a2273a18884cd3d010c Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 7 Jul 2013 09:49:23 +0200 Subject: [PATCH 212/480] Issue #18377: Code cleanup in Python Launcher This changeset fixes a number of compiler warnings in the Python Launcher binary for OSX. It also cleans up whitespace usage in those sources. --- Mac/PythonLauncher/FileSettings.h | 5 -- Mac/PythonLauncher/FileSettings.m | 46 +++++------ Mac/PythonLauncher/MyAppDelegate.m | 6 +- Mac/PythonLauncher/MyDocument.m | 20 ++--- .../PreferencesWindowController.m | 23 +++--- Mac/PythonLauncher/doscript.h | 2 +- Mac/PythonLauncher/doscript.m | 78 +++++++++---------- Mac/PythonLauncher/main.m | 4 +- 8 files changed, 83 insertions(+), 101 deletions(-) diff --git a/Mac/PythonLauncher/FileSettings.h b/Mac/PythonLauncher/FileSettings.h index d807bae59cc..7b74a9b24d2 100755 --- a/Mac/PythonLauncher/FileSettings.h +++ b/Mac/PythonLauncher/FileSettings.h @@ -45,18 +45,13 @@ + (id)getFactorySettingsForFileType: (NSString *)filetype; + (id)newSettingsForFileType: (NSString *)filetype; -//- (id)init; - (id)initForFileType: (NSString *)filetype; - (id)initForFSDefaultFileType: (NSString *)filetype; - (id)initForDefaultFileType: (NSString *)filetype; -//- (id)initWithFileSettings: (FileSettings *)source; - (void)updateFromSource: (id )source; - (NSString *)commandLineForScript: (NSString *)script; -//- (void)applyFactorySettingsForFileType: (NSString *)filetype; -//- (void)saveDefaults; -//- (void)applyUserDefaults: (NSString *)filetype; - (void)applyValuesFromDict: (NSDictionary *)dict; - (void)reset; - (NSArray *) interpreters; diff --git a/Mac/PythonLauncher/FileSettings.m b/Mac/PythonLauncher/FileSettings.m index 66b4fdc0866..3438870b748 100755 --- a/Mac/PythonLauncher/FileSettings.m +++ b/Mac/PythonLauncher/FileSettings.m @@ -14,7 +14,7 @@ + (id)getFactorySettingsForFileType: (NSString *)filetype { static FileSettings *fsdefault_py, *fsdefault_pyw, *fsdefault_pyc; FileSettings **curdefault; - + if ([filetype isEqualToString: @"Python Script"]) { curdefault = &fsdefault_py; } else if ([filetype isEqualToString: @"Python GUI Script"]) { @@ -36,7 +36,7 @@ + (id)getDefaultsForFileType: (NSString *)filetype { static FileSettings *default_py, *default_pyw, *default_pyc; FileSettings **curdefault; - + if ([filetype isEqualToString: @"Python Script"]) { curdefault = &default_py; } else if ([filetype isEqualToString: @"Python GUI Script"]) { @@ -57,7 +57,7 @@ + (id)getDefaultsForFileType: (NSString *)filetype + (id)newSettingsForFileType: (NSString *)filetype { FileSettings *cur; - + cur = [FileSettings new]; [cur initForFileType: filetype]; return [cur retain]; @@ -67,7 +67,7 @@ - (id)initWithFileSettings: (FileSettings *)source { self = [super init]; if (!self) return self; - + interpreter = [source->interpreter retain]; honourhashbang = source->honourhashbang; debug = source->debug; @@ -81,36 +81,30 @@ - (id)initWithFileSettings: (FileSettings *)source with_terminal = source->with_terminal; prefskey = source->prefskey; if (prefskey) [prefskey retain]; - + return self; } - (id)initForFileType: (NSString *)filetype { FileSettings *defaults; - + defaults = [FileSettings getDefaultsForFileType: filetype]; self = [self initWithFileSettings: defaults]; origsource = [defaults retain]; return self; } -//- (id)init -//{ -// self = [self initForFileType: @"Python Script"]; -// return self; -//} - - (id)initForFSDefaultFileType: (NSString *)filetype { int i; NSString *filename; NSDictionary *dict; static NSDictionary *factorySettings; - + self = [super init]; if (!self) return self; - + if (factorySettings == NULL) { NSBundle *bdl = [NSBundle mainBundle]; NSString *path = [ bdl pathForResource: @"factorySettings" @@ -149,18 +143,18 @@ - (void)applyUserDefaults: (NSString *)filetype { NSUserDefaults *defaults; NSDictionary *dict; - + defaults = [NSUserDefaults standardUserDefaults]; dict = [defaults dictionaryForKey: filetype]; if (!dict) return; [self applyValuesFromDict: dict]; } - + - (id)initForDefaultFileType: (NSString *)filetype { FileSettings *fsdefaults; - + fsdefaults = [FileSettings getFactorySettingsForFileType: filetype]; self = [self initWithFileSettings: fsdefaults]; if (!self) return self; @@ -220,7 +214,7 @@ - (void)updateFromSource: (id )source - (void)applyValuesFromDict: (NSDictionary *)dict { id value; - + value = [dict objectForKey: @"interpreter"]; if (value) interpreter = [value retain]; value = [dict objectForKey: @"honourhashbang"]; @@ -247,12 +241,12 @@ - (void)applyValuesFromDict: (NSDictionary *)dict - (NSString*)_replaceSingleQuotes: (NSString*)string { - /* Replace all single-quotes by '"'"', that way shellquoting will - * be correct when the result value is delimited using single quotes. - */ - NSArray* components = [string componentsSeparatedByString:@"'"]; + /* Replace all single-quotes by '"'"', that way shellquoting will + * be correct when the result value is delimited using single quotes. + */ + NSArray* components = [string componentsSeparatedByString:@"'"]; - return [components componentsJoinedByString:@"'\"'\"'"]; + return [components componentsJoinedByString:@"'\"'\"'"]; } - (NSString *)commandLineForScript: (NSString *)script @@ -265,7 +259,7 @@ - (NSString *)commandLineForScript: (NSString *)script script_dir = [script substringToIndex: [script length]-[[script lastPathComponent] length]]; - + if (honourhashbang && (fp=fopen([script fileSystemRepresentation], "r")) && fgets(hashbangbuf, sizeof(hashbangbuf), fp) && @@ -278,7 +272,7 @@ - (NSString *)commandLineForScript: (NSString *)script } if (!cur_interp) cur_interp = interpreter; - + return [NSString stringWithFormat: @"cd '%@' && '%@'%s%s%s%s%s%s %@ '%@' %@ %s", [self _replaceSingleQuotes:script_dir], @@ -297,7 +291,7 @@ - (NSString *)commandLineForScript: (NSString *)script - (NSArray *) interpreters { return interpreters;}; -// FileSettingsSource protocol +// FileSettingsSource protocol - (NSString *) interpreter { return interpreter;}; - (BOOL) honourhashbang { return honourhashbang; }; - (BOOL) debug { return debug;}; diff --git a/Mac/PythonLauncher/MyAppDelegate.m b/Mac/PythonLauncher/MyAppDelegate.m index a5ba7510784..e75fb06616b 100644 --- a/Mac/PythonLauncher/MyAppDelegate.m +++ b/Mac/PythonLauncher/MyAppDelegate.m @@ -33,7 +33,7 @@ - (void)applicationDidFinishLaunching:(NSNotification *)notification - (BOOL)shouldShowUI { - // if this call comes before applicationDidFinishLaunching: we + // if this call comes before applicationDidFinishLaunching: we // should terminate immedeately after starting the script. if (!initial_action_done) should_terminate = YES; @@ -62,7 +62,7 @@ - (void)testFileTypeBinding static NSString *extensions[] = { @"py", @"pyw", @"pyc", NULL}; NSString **ext_p; int i; - + if ([[NSUserDefaults standardUserDefaults] boolForKey: @"SkipFileBindingTest"]) return; ourUrl = [NSURL fileURLWithPath: [[NSBundle mainBundle] bundlePath]]; @@ -92,5 +92,5 @@ - (void)testFileTypeBinding } } } - + @end diff --git a/Mac/PythonLauncher/MyDocument.m b/Mac/PythonLauncher/MyDocument.m index 8f851e74e30..90c5db9dc5b 100755 --- a/Mac/PythonLauncher/MyDocument.m +++ b/Mac/PythonLauncher/MyDocument.m @@ -16,7 +16,7 @@ - (id)init { self = [super init]; if (self) { - + // Add your subclass-specific initialization here. // If an error occurs here, send a [self dealloc] message and return nil. script = [@".py" retain]; @@ -37,20 +37,17 @@ - (void)close { NSApplication *app = [NSApplication sharedApplication]; [super close]; - if ([[app delegate] shouldTerminate]) + if ([(MyAppDelegate*)[app delegate] shouldTerminate]) [app terminate: self]; } - (void)load_defaults { -// if (settings) [settings release]; settings = [FileSettings newSettingsForFileType: filetype]; } - (void)update_display { -// [[self window] setTitle: script]; - [interpreter setStringValue: [settings interpreter]]; [honourhashbang setState: [settings honourhashbang]]; [debug setState: [settings debug]]; @@ -62,7 +59,7 @@ - (void)update_display [others setStringValue: [settings others]]; [scriptargs setStringValue: [settings scriptargs]]; [with_terminal setState: [settings with_terminal]]; - + [commandline setStringValue: [settings commandLineForScript: script]]; } @@ -75,7 +72,7 @@ - (BOOL)run { const char *cmdline; int sts; - + cmdline = [[settings commandLineForScript: script] UTF8String]; if ([settings with_terminal]) { sts = doscript(cmdline); @@ -107,14 +104,13 @@ - (BOOL)readFromFile:(NSString *)fileName ofType:(NSString *)type; { // Insert code here to read your document from the given data. You can also choose to override -loadFileWrapperRepresentation:ofType: or -readFromFile:ofType: instead. BOOL show_ui; - - // ask the app delegate whether we should show the UI or not. - show_ui = [[[NSApplication sharedApplication] delegate] shouldShowUI]; + + // ask the app delegate whether we should show the UI or not. + show_ui = [(MyAppDelegate*)[[NSApplication sharedApplication] delegate] shouldShowUI]; [script release]; script = [fileName retain]; [filetype release]; filetype = [type retain]; -// if (settings) [settings release]; settings = [FileSettings newSettingsForFileType: filetype]; if (show_ui) { [self update_display]; @@ -152,7 +148,7 @@ - (IBAction)do_apply:(id)sender [self update_display]; } -// FileSettingsSource protocol +// FileSettingsSource protocol - (NSString *) interpreter { return [interpreter stringValue];}; - (BOOL) honourhashbang { return [honourhashbang state];}; - (BOOL) debug { return [debug state];}; diff --git a/Mac/PythonLauncher/PreferencesWindowController.m b/Mac/PythonLauncher/PreferencesWindowController.m index 311c37502bb..ec5bbe84c8c 100644 --- a/Mac/PythonLauncher/PreferencesWindowController.m +++ b/Mac/PythonLauncher/PreferencesWindowController.m @@ -5,7 +5,7 @@ @implementation PreferencesWindowController + getPreferencesWindow { static PreferencesWindowController *_singleton; - + if (!_singleton) _singleton = [[PreferencesWindowController alloc] init]; [_singleton showWindow: _singleton]; @@ -21,15 +21,13 @@ - (id) init - (void)load_defaults { NSString *title = [filetype titleOfSelectedItem]; - + settings = [FileSettings getDefaultsForFileType: title]; } - (void)update_display { -// [[self window] setTitle: script]; - - [interpreter reloadData]; + [interpreter reloadData]; [interpreter setStringValue: [settings interpreter]]; [honourhashbang setState: [settings honourhashbang]]; [debug setState: [settings debug]]; @@ -41,7 +39,6 @@ - (void)update_display [others setStringValue: [settings others]]; [with_terminal setState: [settings with_terminal]]; // Not scriptargs, it isn't for preferences - [commandline setStringValue: [settings commandLineForScript: @""]]; } @@ -75,7 +72,7 @@ - (IBAction)do_apply:(id)sender [self update_display]; } -// FileSettingsSource protocol +// FileSettingsSource protocol - (NSString *) interpreter { return [interpreter stringValue];}; - (BOOL) honourhashbang { return [honourhashbang state]; }; - (BOOL) debug { return [debug state];}; @@ -98,23 +95,23 @@ - (void)controlTextDidChange:(NSNotification *)aNotification // NSComboBoxDataSource protocol - (unsigned int)comboBox:(NSComboBox *)aComboBox indexOfItemWithStringValue:(NSString *)aString { - NSArray *interp_list = [settings interpreters]; + NSArray *interp_list = [settings interpreters]; unsigned int rv = [interp_list indexOfObjectIdenticalTo: aString]; - return rv; + return rv; } - (id)comboBox:(NSComboBox *)aComboBox objectValueForItemAtIndex:(int)index { - NSArray *interp_list = [settings interpreters]; + NSArray *interp_list = [settings interpreters]; id rv = [interp_list objectAtIndex: index]; - return rv; + return rv; } - (int)numberOfItemsInComboBox:(NSComboBox *)aComboBox { - NSArray *interp_list = [settings interpreters]; + NSArray *interp_list = [settings interpreters]; int rv = [interp_list count]; - return rv; + return rv; } diff --git a/Mac/PythonLauncher/doscript.h b/Mac/PythonLauncher/doscript.h index eef0b5641f6..3fd3187cd4a 100644 --- a/Mac/PythonLauncher/doscript.h +++ b/Mac/PythonLauncher/doscript.h @@ -9,4 +9,4 @@ #include -extern int doscript(const char *command); \ No newline at end of file +extern int doscript(const char *command); diff --git a/Mac/PythonLauncher/doscript.m b/Mac/PythonLauncher/doscript.m index 024b883f190..cbb783ba3e8 100644 --- a/Mac/PythonLauncher/doscript.m +++ b/Mac/PythonLauncher/doscript.m @@ -11,49 +11,49 @@ #import #import "doscript.h" -extern int +extern int doscript(const char *command) { - char *bundleID = "com.apple.Terminal"; - AppleEvent evt, res; - AEDesc desc; - OSStatus err; + char *bundleID = "com.apple.Terminal"; + AppleEvent evt, res; + AEDesc desc; + OSStatus err; - [[NSWorkspace sharedWorkspace] launchApplication:@"/Applications/Utilities/Terminal.app/"]; + [[NSWorkspace sharedWorkspace] launchApplication:@"/Applications/Utilities/Terminal.app/"]; - // Build event - err = AEBuildAppleEvent(kAECoreSuite, kAEDoScript, - typeApplicationBundleID, - bundleID, strlen(bundleID), - kAutoGenerateReturnID, - kAnyTransactionID, - &evt, NULL, - "'----':utf8(@)", strlen(command), - command); - if (err) { - NSLog(@"AEBuildAppleEvent failed: %d\n", err); - return err; - } + // Build event + err = AEBuildAppleEvent(kAECoreSuite, kAEDoScript, + typeApplicationBundleID, + bundleID, strlen(bundleID), + kAutoGenerateReturnID, + kAnyTransactionID, + &evt, NULL, + "'----':utf8(@)", strlen(command), + command); + if (err) { + NSLog(@"AEBuildAppleEvent failed: %ld\n", (long)err); + return err; + } - // Send event and check for any Apple Event Manager errors - err = AESendMessage(&evt, &res, kAEWaitReply, kAEDefaultTimeout); - AEDisposeDesc(&evt); - if (err) { - NSLog(@"AESendMessage failed: %d\n", err); - return err; - } - // Check for any application errors - err = AEGetParamDesc(&res, keyErrorNumber, typeSInt32, &desc); - AEDisposeDesc(&res); - if (!err) { - AEGetDescData(&desc, &err, sizeof(err)); - NSLog(@"Terminal returned an error: %d", err); - AEDisposeDesc(&desc); - } else if (err == errAEDescNotFound) { - err = noErr; - } else { - NSLog(@"AEGetPArmDesc returned an error: %d", err); - } + // Send event and check for any Apple Event Manager errors + err = AESendMessage(&evt, &res, kAEWaitReply, kAEDefaultTimeout); + AEDisposeDesc(&evt); + if (err) { + NSLog(@"AESendMessage failed: %ld\n", (long)err); + return err; + } + // Check for any application errors + err = AEGetParamDesc(&res, keyErrorNumber, typeSInt32, &desc); + AEDisposeDesc(&res); + if (!err) { + AEGetDescData(&desc, &err, sizeof(err)); + NSLog(@"Terminal returned an error: %ld", (long)err); + AEDisposeDesc(&desc); + } else if (err == errAEDescNotFound) { + err = noErr; + } else { + NSLog(@"AEGetPArmDesc returned an error: %ld", (long)err); + } - return err; + return err; } diff --git a/Mac/PythonLauncher/main.m b/Mac/PythonLauncher/main.m index 6841433f429..04b4d7321f7 100755 --- a/Mac/PythonLauncher/main.m +++ b/Mac/PythonLauncher/main.m @@ -11,7 +11,7 @@ int main(int argc, const char *argv[]) { - char *home = getenv("HOME"); - if (home) chdir(home); + char *home = getenv("HOME"); + if (home) chdir(home); return NSApplicationMain(argc, argv); } From 41e8a4b760c4628cb85701d0b88e8496b664374d Mon Sep 17 00:00:00 2001 From: R David Murray Date: Wed, 10 Jul 2013 16:23:15 -0400 Subject: [PATCH 213/480] #18424: PEP8ify the tense of the sum docstring. --- Python/bltinmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index cdc908084fc..d22dca2f487 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2434,9 +2434,9 @@ builtin_sum(PyObject *self, PyObject *args) PyDoc_STRVAR(sum_doc, "sum(sequence[, start]) -> value\n\ \n\ -Returns the sum of a sequence of numbers (NOT strings) plus the value\n\ +Return the sum of a sequence of numbers (NOT strings) plus the value\n\ of parameter 'start' (which defaults to 0). When the sequence is\n\ -empty, returns start."); +empty, return start."); static PyObject * From 1f41fbf3c3118b7955ced84ad8712d24299a0eed Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 11 Jul 2013 13:33:55 +0200 Subject: [PATCH 214/480] Issue #18427: str.replace could crash the interpreter with huge strings. This fixes two places where 'int' was used to represent the size of strings, instead of 'Py_ssize_t'. (The issue is not present in the corresponding code in the 3.x branches) Fixes #18427 --- Misc/NEWS | 4 +++- Objects/stringobject.c | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index c3689e93188..804e8c2125d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,6 +24,8 @@ Core and Builtins Library ------- +- Issue #18427: str.replace could crash the interpreter with huge strings. + - Issue #18347: ElementTree's html serializer now preserves the case of closing tags. @@ -88,7 +90,7 @@ IDLE - Issue #7136: In the Idle File menu, "New Window" is renamed "New File". Patch by Tal Einat, Roget Serwy, and Todd Rovito. - + - Issue #8515: Set __file__ when run file in IDLE. Initial patch by Bruce Frederiksen. diff --git a/Objects/stringobject.c b/Objects/stringobject.c index 12091973779..b80ef87b0d8 100644 --- a/Objects/stringobject.c +++ b/Objects/stringobject.c @@ -882,9 +882,9 @@ string_print(PyStringObject *op, FILE *fp, int flags) size -= chunk_size; } #ifdef __VMS - if (size) fwrite(data, (int)size, 1, fp); + if (size) fwrite(data, (size_t)size, 1, fp); #else - fwrite(data, 1, (int)size, fp); + fwrite(data, 1, (size_t)size, fp); #endif Py_END_ALLOW_THREADS return 0; @@ -2332,7 +2332,7 @@ return_self(PyStringObject *self) } Py_LOCAL_INLINE(Py_ssize_t) -countchar(const char *target, int target_len, char c, Py_ssize_t maxcount) +countchar(const char *target, Py_ssize_t target_len, char c, Py_ssize_t maxcount) { Py_ssize_t count=0; const char *start=target; From f188e090e37115da0c726bea169ecb7648111a08 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Jul 2013 18:25:19 +0300 Subject: [PATCH 215/480] Issue #18336. Fix a link to StreamReader's read() method. --- Doc/library/codecs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 39a3c5dff60..61da3ed9e7e 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -653,7 +653,7 @@ compatible with the Python codec registry. Read one line from the input stream and return the decoded data. *size*, if given, is passed as size argument to the stream's - :meth:`readline` method. + :meth:`read` method. If *keepends* is false line-endings will be stripped from the lines returned. From 4b8077f02a9d33d05d360678f8a706ee38343ed7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Jul 2013 19:14:07 +0300 Subject: [PATCH 216/480] Fix reference leaks introduced by the patch for issue #5308. --- Python/marshal.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Python/marshal.c b/Python/marshal.c index 33685b9d7c2..6b285aaa41e 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -88,7 +88,7 @@ w_more(int c, WFILE *p) } static void -w_string(char *s, Py_ssize_t n, WFILE *p) +w_string(const char *s, Py_ssize_t n, WFILE *p) { if (p->fp != NULL) { fwrite(s, 1, n, p->fp); @@ -141,6 +141,13 @@ w_long64(long x, WFILE *p) # define W_SIZE w_long #endif +static void +w_pstring(const char *s, Py_ssize_t n, WFILE *p) +{ + W_SIZE(n, p); + w_string(s, n, p); +} + /* We assume that Python longs are stored internally in base some power of 2**15; for the sake of portability we'll always read and write them in base exactly 2**15. */ @@ -338,9 +345,7 @@ w_object(PyObject *v, WFILE *p) else { w_byte(TYPE_STRING, p); } - n = PyString_GET_SIZE(v); - W_SIZE(n, p); - w_string(PyString_AS_STRING(v), n, p); + w_pstring(PyBytes_AS_STRING(v), PyString_GET_SIZE(v), p); } #ifdef Py_USING_UNICODE else if (PyUnicode_CheckExact(v)) { @@ -352,9 +357,7 @@ w_object(PyObject *v, WFILE *p) return; } w_byte(TYPE_UNICODE, p); - n = PyString_GET_SIZE(utf8); - W_SIZE(n, p); - w_string(PyString_AS_STRING(utf8), n, p); + w_pstring(PyString_AS_STRING(utf8), PyString_GET_SIZE(utf8), p); Py_DECREF(utf8); } #endif @@ -441,8 +444,7 @@ w_object(PyObject *v, WFILE *p) PyBufferProcs *pb = v->ob_type->tp_as_buffer; w_byte(TYPE_STRING, p); n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s); - W_SIZE(n, p); - w_string(s, n, p); + w_pstring(s, n, p); } else { w_byte(TYPE_UNKNOWN, p); From f0576059835000e3330be35812abf182d116e2e7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Jul 2013 20:32:48 +0300 Subject: [PATCH 217/480] Issue #18101: Tcl.split() now process Unicode strings nested in a tuple as it do with byte strings. Added tests for Tcl.split() and tcl.splitline(). --- Lib/test/test_tcl.py | 60 ++++++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Modules/_tkinter.c | 27 ++++++++++++++++++++ 3 files changed, 90 insertions(+) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index a0e9e627ac6..3da87d9b253 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -184,6 +184,66 @@ def passValue(value): self.assertEqual(passValue(f), f) self.assertEqual(passValue((1, '2', (3.4,))), (1, '2', (3.4,))) + def test_splitlist(self): + splitlist = self.interp.tk.splitlist + call = self.interp.tk.call + self.assertRaises(TypeError, splitlist) + self.assertRaises(TypeError, splitlist, 'a', 'b') + self.assertRaises(TypeError, splitlist, 2) + testcases = [ + ('2', ('2',)), + ('', ()), + ('{}', ('',)), + ('""', ('',)), + ('a\n b\t\r c\n ', ('a', 'b', 'c')), + (u'a\n b\t\r c\n ', ('a', 'b', 'c')), + ('a \xe2\x82\xac', ('a', '\xe2\x82\xac')), + (u'a \u20ac', ('a', '\xe2\x82\xac')), + ('a {b c}', ('a', 'b c')), + (r'a b\ c', ('a', 'b c')), + (('a', 'b c'), ('a', 'b c')), + ('a 2', ('a', '2')), + (('a', 2), ('a', 2)), + ('a 3.4', ('a', '3.4')), + (('a', 3.4), ('a', 3.4)), + ((), ()), + (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), + ] + for arg, res in testcases: + self.assertEqual(splitlist(arg), res) + self.assertRaises(TclError, splitlist, '{') + + def test_split(self): + split = self.interp.tk.split + call = self.interp.tk.call + self.assertRaises(TypeError, split) + self.assertRaises(TypeError, split, 'a', 'b') + self.assertRaises(TypeError, split, 2) + testcases = [ + ('2', '2'), + ('', ''), + ('{}', ''), + ('""', ''), + ('{', '{'), + ('a\n b\t\r c\n ', ('a', 'b', 'c')), + (u'a\n b\t\r c\n ', ('a', 'b', 'c')), + ('a \xe2\x82\xac', ('a', '\xe2\x82\xac')), + (u'a \u20ac', ('a', '\xe2\x82\xac')), + ('a {b c}', ('a', ('b', 'c'))), + (r'a b\ c', ('a', ('b', 'c'))), + (('a', 'b c'), ('a', ('b', 'c'))), + (('a', u'b c'), ('a', ('b', 'c'))), + ('a 2', ('a', '2')), + (('a', 2), ('a', 2)), + ('a 3.4', ('a', '3.4')), + (('a', 3.4), ('a', 3.4)), + (('a', (2, 3.4)), ('a', (2, 3.4))), + ((), ()), + (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), + ] + for arg, res in testcases: + self.assertEqual(split(arg), res) + def test_main(): test_support.run_unittest(TclTest, TkinterTest) diff --git a/Misc/NEWS b/Misc/NEWS index 804e8c2125d..9bd57913b4d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,6 +24,9 @@ Core and Builtins Library ------- +- Issue #18101: Tcl.split() now process Unicode strings nested in a tuple as it + do with byte strings. + - Issue #18427: str.replace could crash the interpreter with huge strings. - Issue #18347: ElementTree's html serializer now preserves the case of diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 7872df3bf18..aed42ac3e4b 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -547,6 +547,33 @@ SplitObj(PyObject *arg) return Split(PyString_AsString(arg)); /* Fall through, returning arg. */ } + else if (PyUnicode_Check(arg)) { + int argc; + char **argv; + char *list; + PyObject *s = PyUnicode_AsUTF8String(arg); + + if (s == NULL) { + Py_INCREF(arg); + return arg; + } + list = PyString_AsString(s); + + if (list == NULL || + Tcl_SplitList((Tcl_Interp *)NULL, list, &argc, &argv) != TCL_OK) { + Py_DECREF(s); + Py_INCREF(arg); + return arg; + } + Tcl_Free(FREECAST argv); + if (argc > 1) { + PyObject *v = Split(list); + Py_DECREF(s); + return v; + } + Py_DECREF(s); + /* Fall through, returning arg. */ + } Py_INCREF(arg); return arg; } From ba5971d59f01cecd4255c4b808887b56fe7b6e48 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Fri, 12 Jul 2013 18:21:41 -0400 Subject: [PATCH 218/480] #18389: Clarify that relpath does not access the file system. Initial patch by Madison May. --- Doc/library/os.path.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 5e245d0df5b..c84aaf29e10 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -236,8 +236,10 @@ the :mod:`glob` module.) .. function:: relpath(path[, start]) - Return a relative filepath to *path* either from the current directory or from - an optional *start* point. + Return a relative filepath to *path* either from the current directory or + from an optional *start* directory. This is a path computation: the + filesystem is not accessed to confirm the existence or nature of *path* or + *start*. *start* defaults to :attr:`os.curdir`. From 8b627877bb88117416b0b7b604917f558cdf1caf Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 12 Jul 2013 20:10:17 -0400 Subject: [PATCH 219/480] Issue #18365: Add mock Text class and test thereof versus tk.Text. Based on patches by Todd.Rovito and Phil Webster. --- Lib/idlelib/idle_test/mock_tk.py | 226 +++++++++++++++++++++++++++- Lib/idlelib/idle_test/test_text.py | 228 +++++++++++++++++++++++++++++ 2 files changed, 449 insertions(+), 5 deletions(-) create mode 100644 Lib/idlelib/idle_test/test_text.py diff --git a/Lib/idlelib/idle_test/mock_tk.py b/Lib/idlelib/idle_test/mock_tk.py index 60724a18c39..a12db134287 100644 --- a/Lib/idlelib/idle_test/mock_tk.py +++ b/Lib/idlelib/idle_test/mock_tk.py @@ -1,4 +1,5 @@ """Classes that replace tkinter gui objects used by an object being tested. + A gui object is anything with a master or parent paramenter, which is typically required in spite of what the doc strings say. """ @@ -15,8 +16,10 @@ def get(self): return self.value class Mbox_func(object): - """Generic mock for messagebox functions. All have same call signature. - Mbox instantiates once for each function. Tester uses attributes. + """Generic mock for messagebox functions, which all have the same signature. + + Instead of displaying a message box, the mock's call method saves the + arguments as instance attributes, which test functions can then examime. """ def __init__(self): self.result = None # The return for all show funcs @@ -30,6 +33,7 @@ def __call__(self, title, message, *args, **kwds): class Mbox(object): """Mock for tkinter.messagebox with an Mbox_func for each function. + This module was 'tkMessageBox' in 2.x; hence the 'import as' in 3.x. Example usage in test_module.py for testing functios in module.py: --- @@ -49,9 +53,9 @@ def setUpClass(cls): def tearDownClass(cls): module.tkMessageBox = orig_mbox --- - When tkMessageBox functions are the only gui making calls in a method, - this replacement makes the method gui-free and unit-testable. - For 'ask' functions, set func.result return before calling method. + For 'ask' functions, set func.result return value before calling the method + that uses the message function. When tkMessageBox functions are the + only gui alls in a method, this replacement makes the method gui-free, """ askokcancel = Mbox_func() # True or False askquestion = Mbox_func() # 'yes' or 'no' @@ -61,3 +65,215 @@ def tearDownClass(cls): showerror = Mbox_func() # None showinfo = Mbox_func() # None showwarning = Mbox_func() # None + +from _tkinter import TclError + +class Text(object): + """A semi-functional non-gui replacement for tkinter.Text text editors. + + The mock's data model is that a text is a list of \n-terminated lines. + The mock adds an empty string at the beginning of the list so that the + index of actual lines start at 1, as with Tk. The methods never see this. + Tk initializes files with a terminal \n that cannot be deleted. It is + invisible in the sense that one cannot move the cursor beyond it. + + This class is only tested (and valid) with strings of ascii chars. + For testing, we are not concerned with Tk Text's treatment of, + for instance, 0-width characters or character + accent. + """ + def __init__(self, master=None, cnf={}, **kw): + '''Initialize mock, non-gui, text-only Text widget. + + At present, all args are ignored. Almost all affect visual behavior. + There are just a few Text-only options that affect text behavior. + ''' + self.data = ['', '\n'] + + def index(self, index): + "Return string version of index decoded according to current text." + return "%s.%s" % self._decode(index, endflag=1) + + def _decode(self, index, endflag=0): + """Return a (line, char) tuple of int indexes into self.data. + + This implements .index without converting the result back to a string. + The result is contrained by the number of lines and linelengths of + self.data. For many indexes, the result is initally (1, 0). + + The input index may have any of several possible forms: + * line.char float: converted to 'line.char' string; + * 'line.char' string, where line and char are decimal integers; + * 'line.char lineend', where lineend='lineend' (and char is ignored); + * 'line.end', where end='end' (same as above); + * 'insert', the positions before terminal \n; + * 'end', whose meaning depends on the endflag passed to ._endex. + * 'sel.first' or 'sel.last', where sel is a tag -- not implemented. + """ + if isinstance(index, (float, bytes)): + index = str(index) + try: + index=index.lower() + except AttributeError: + raise TclError('bad text index "%s"' % index) from None + + lastline = len(self.data) - 1 # same as number of text lines + if index == 'insert': + return lastline, len(self.data[lastline]) - 1 + elif index == 'end': + return self._endex(endflag) + + line, char = index.split('.') + line = int(line) + + # Out of bounds line becomes first or last ('end') index + if line < 1: + return 1, 0 + elif line > lastline: + return self._endex(endflag) + + linelength = len(self.data[line]) -1 # position before/at \n + if char.endswith(' lineend') or char == 'end': + return line, linelength + # Tk requires that ignored chars before ' lineend' be valid int + + # Out of bounds char becomes first or last index of line + char = int(char) + if char < 0: + char = 0 + elif char > linelength: + char = linelength + return line, char + + def _endex(self, endflag): + '''Return position for 'end' or line overflow corresponding to endflag. + + -1: position before terminal \n; for .insert(), .delete + 0: position after terminal \n; for .get, .delete index 1 + 1: same viewed as begininning of non-existent next line (for .index) + ''' + n = len(self.data) + if endflag == 1: + return n, 0 + else: + n -= 1 + return n, len(self.data[n]) + endflag + + + def insert(self, index, chars): + "Insert chars before the character at index." + + if not chars: # ''.splitlines() is [], not [''] + return + chars = chars.splitlines(True) + if chars[-1][-1] == '\n': + chars.append('') + line, char = self._decode(index, -1) + before = self.data[line][:char] + after = self.data[line][char:] + self.data[line] = before + chars[0] + self.data[line+1:line+1] = chars[1:] + self.data[line+len(chars)-1] += after + + + def get(self, index1, index2=None): + "Return slice from index1 to index2 (default is 'index1+1')." + + startline, startchar = self._decode(index1) + if index2 is None: + endline, endchar = startline, startchar+1 + else: + endline, endchar = self._decode(index2) + + if startline == endline: + return self.data[startline][startchar:endchar] + else: + lines = [self.data[startline][startchar:]] + for i in range(startline+1, endline): + lines.append(self.data[i]) + lines.append(self.data[endline][:endchar]) + return ''.join(lines) + + + def delete(self, index1, index2=None): + '''Delete slice from index1 to index2 (default is 'index1+1'). + + Adjust default index2 ('index+1) for line ends. + Do not delete the terminal \n at the very end of self.data ([-1][-1]). + ''' + startline, startchar = self._decode(index1, -1) + if index2 is None: + if startchar < len(self.data[startline])-1: + # not deleting \n + endline, endchar = startline, startchar+1 + elif startline < len(self.data) - 1: + # deleting non-terminal \n, convert 'index1+1 to start of next line + endline, endchar = startline+1, 0 + else: + # do not delete terminal \n if index1 == 'insert' + return + else: + endline, endchar = self._decode(index2, -1) + # restricting end position to insert position excludes terminal \n + + if startline == endline and startchar < endchar: + self.data[startline] = self.data[startline][:startchar] + \ + self.data[startline][endchar:] + elif startline < endline: + self.data[startline] = self.data[startline][:startchar] + \ + self.data[endline][endchar:] + startline += 1 + for i in range(startline, endline+1): + del self.data[startline] + + def compare(self, index1, op, index2): + line1, char1 = self._decode(index1) + line2, char2 = self._decode(index2) + if op == '<': + return line1 < line2 or line1 == line2 and char1 < char2 + elif op == '<=': + return line1 < line2 or line1 == line2 and char1 <= char2 + elif op == '>': + return line1 > line2 or line1 == line2 and char1 > char2 + elif op == '>=': + return line1 > line2 or line1 == line2 and char1 >= char2 + elif op == '==': + return line1 == line2 and char1 == char2 + elif op == '!=': + return line1 != line2 or char1 != char2 + else: + raise TclError('''bad comparison operator "%s":''' + '''must be <, <=, ==, >=, >, or !=''' % op) + + # The following Text methods normally do something and return None. + # Whether doing nothing is sufficient for a test will depend on the test. + + def mark_set(self, name, index): + "Set mark *name* before the character at index." + pass + + def mark_unset(self, *markNames): + "Delete all marks in markNames." + + def tag_remove(self, tagName, index1, index2=None): + "Remove tag tagName from all characters between index1 and index2." + pass + + # The following Text methods affect the graphics screen and return None. + # Doing nothing should always be sufficient for tests. + + def scan_dragto(self, x, y): + "Adjust the view of the text according to scan_mark" + + def scan_mark(self, x, y): + "Remember the current X, Y coordinates." + + def see(self, index): + "Scroll screen to make the character at INDEX is visible." + pass + + # The following is a Misc method inheritet by Text. + # It should properly go in a Misc mock, but is included here for now. + + def bind(sequence=None, func=None, add=None): + "Bind to this widget at event sequence a call to function func." + pass diff --git a/Lib/idlelib/idle_test/test_text.py b/Lib/idlelib/idle_test/test_text.py new file mode 100644 index 00000000000..69419c1b0ca --- /dev/null +++ b/Lib/idlelib/idle_test/test_text.py @@ -0,0 +1,228 @@ +# Test mock_tk.Text class against tkinter.Text class by running same tests with both. +import unittest +from test.support import requires + +from _tkinter import TclError +import tkinter as tk + +class TextTest(object): + + hw = 'hello\nworld' # usual initial insert after initialization + hwn = hw+'\n' # \n present at initialization, before insert + + Text = None + def setUp(self): + self.text = self.Text() + + def test_init(self): + self.assertEqual(self.text.get('1.0'), '\n') + self.assertEqual(self.text.get('end'), '') + + def test_index_empty(self): + index = self.text.index + + for dex in (-1.0, 0.3, '1.-1', '1.0', '1.0 lineend', '1.end', '1.33', + 'insert'): + self.assertEqual(index(dex), '1.0') + + for dex in 'end', 2.0, '2.1', '33.44': + self.assertEqual(index(dex), '2.0') + + def test_index_data(self): + index = self.text.index + self.text.insert('1.0', self.hw) + + for dex in -1.0, 0.3, '1.-1', '1.0': + self.assertEqual(index(dex), '1.0') + + for dex in '1.0 lineend', '1.end', '1.33': + self.assertEqual(index(dex), '1.5') + + for dex in 'end', '33.44': + self.assertEqual(index(dex), '3.0') + + def test_get(self): + get = self.text.get + Equal = self.assertEqual + self.text.insert('1.0', self.hw) + + Equal(get('end'), '') + Equal(get('end', 'end'), '') + Equal(get('1.0'), 'h') + Equal(get('1.0', '1.1'), 'h') + Equal(get('1.0', '1.3'), 'hel') + Equal(get('1.1', '1.3'), 'el') + Equal(get('1.0', '1.0 lineend'), 'hello') + Equal(get('1.0', '1.10'), 'hello') + Equal(get('1.0 lineend'), '\n') + Equal(get('1.1', '2.3'), 'ello\nwor') + Equal(get('1.0', '2.5'), self.hw) + Equal(get('1.0', 'end'), self.hwn) + Equal(get('0.0', '5.0'), self.hwn) + + def test_insert(self): + insert = self.text.insert + get = self.text.get + Equal = self.assertEqual + + insert('1.0', self.hw) + Equal(get('1.0', 'end'), self.hwn) + + insert('1.0', '') # nothing + Equal(get('1.0', 'end'), self.hwn) + + insert('1.0', '*') + Equal(get('1.0', 'end'), '*hello\nworld\n') + + insert('1.0 lineend', '*') + Equal(get('1.0', 'end'), '*hello*\nworld\n') + + insert('2.3', '*') + Equal(get('1.0', 'end'), '*hello*\nwor*ld\n') + + insert('end', 'x') + Equal(get('1.0', 'end'), '*hello*\nwor*ldx\n') + + insert('1.4', 'x\n') + Equal(get('1.0', 'end'), '*helx\nlo*\nwor*ldx\n') + + def test_no_delete(self): + # if index1 == 'insert' or 'end' or >= end, there is no deletion + delete = self.text.delete + get = self.text.get + Equal = self.assertEqual + self.text.insert('1.0', self.hw) + + delete('insert') + Equal(get('1.0', 'end'), self.hwn) + + delete('end') + Equal(get('1.0', 'end'), self.hwn) + + delete('insert', 'end') + Equal(get('1.0', 'end'), self.hwn) + + delete('insert', '5.5') + Equal(get('1.0', 'end'), self.hwn) + + delete('1.4', '1.0') + Equal(get('1.0', 'end'), self.hwn) + + delete('1.4', '1.4') + Equal(get('1.0', 'end'), self.hwn) + + def test_delete_char(self): + delete = self.text.delete + get = self.text.get + Equal = self.assertEqual + self.text.insert('1.0', self.hw) + + delete('1.0') + Equal(get('1.0', '1.end'), 'ello') + + delete('1.0', '1.1') + Equal(get('1.0', '1.end'), 'llo') + + # delete \n and combine 2 lines into 1 + delete('1.end') + Equal(get('1.0', '1.end'), 'lloworld') + + self.text.insert('1.3', '\n') + delete('1.10') + Equal(get('1.0', '1.end'), 'lloworld') + + self.text.insert('1.3', '\n') + delete('1.3', '2.0') + Equal(get('1.0', '1.end'), 'lloworld') + + def test_delete_slice(self): + delete = self.text.delete + get = self.text.get + Equal = self.assertEqual + self.text.insert('1.0', self.hw) + + delete('1.0', '1.0 lineend') + Equal(get('1.0', 'end'), '\nworld\n') + + delete('1.0', 'end') + Equal(get('1.0', 'end'), '\n') + + self.text.insert('1.0', self.hw) + delete('1.0', '2.0') + Equal(get('1.0', 'end'), 'world\n') + + delete('1.0', 'end') + Equal(get('1.0', 'end'), '\n') + + self.text.insert('1.0', self.hw) + delete('1.2', '2.3') + Equal(get('1.0', 'end'), 'held\n') + + def test_multiple_lines(self): # insert and delete + self.text.insert('1.0', 'hello') + + self.text.insert('1.3', '1\n2\n3\n4\n5') + self.assertEqual(self.text.get('1.0', 'end'), 'hel1\n2\n3\n4\n5lo\n') + + self.text.delete('1.3', '5.1') + self.assertEqual(self.text.get('1.0', 'end'), 'hello\n') + + def test_compare(self): + compare = self.text.compare + Equal = self.assertEqual + # need data so indexes not squished to 1,0 + self.text.insert('1.0', 'First\nSecond\nThird\n') + + self.assertRaises(TclError, compare, '2.2', 'op', '2.2') + + for op, less1, less0, equal, greater0, greater1 in ( + ('<', True, True, False, False, False), + ('<=', True, True, True, False, False), + ('>', False, False, False, True, True), + ('>=', False, False, True, True, True), + ('==', False, False, True, False, False), + ('!=', True, True, False, True, True), + ): + Equal(compare('1.1', op, '2.2'), less1, op) + Equal(compare('2.1', op, '2.2'), less0, op) + Equal(compare('2.2', op, '2.2'), equal, op) + Equal(compare('2.3', op, '2.2'), greater0, op) + Equal(compare('3.3', op, '2.2'), greater1, op) + + +class MockTextTest(TextTest, unittest.TestCase): + + @classmethod + def setUpClass(cls): + from idlelib.idle_test.mock_tk import Text + cls.Text = Text + + def test_decode(self): + # test endflags (-1, 0) not tested by test_index (which uses +1) + decode = self.text._decode + Equal = self.assertEqual + self.text.insert('1.0', self.hw) + + Equal(decode('end', -1), (2, 5)) + Equal(decode('3.1', -1), (2, 5)) + Equal(decode('end', 0), (2, 6)) + Equal(decode('3.1', 0), (2, 6)) + + +class TkTextTest(TextTest, unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + from tkinter import Tk, Text + cls.Text = Text + cls.root = Tk() + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) + From 39d43ff0bae650df21b56b0355008117224d1042 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 12 Jul 2013 20:18:33 -0400 Subject: [PATCH 220/480] Issue #18365: normalize whitespace --- Lib/idlelib/idle_test/mock_tk.py | 12 ++++++------ Lib/idlelib/idle_test/test_text.py | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Lib/idlelib/idle_test/mock_tk.py b/Lib/idlelib/idle_test/mock_tk.py index a12db134287..279f4bef242 100644 --- a/Lib/idlelib/idle_test/mock_tk.py +++ b/Lib/idlelib/idle_test/mock_tk.py @@ -210,7 +210,7 @@ def delete(self, index1, index2=None): endline, endchar = startline+1, 0 else: # do not delete terminal \n if index1 == 'insert' - return + return else: endline, endchar = self._decode(index2, -1) # restricting end position to insert position excludes terminal \n @@ -219,11 +219,11 @@ def delete(self, index1, index2=None): self.data[startline] = self.data[startline][:startchar] + \ self.data[startline][endchar:] elif startline < endline: - self.data[startline] = self.data[startline][:startchar] + \ - self.data[endline][endchar:] - startline += 1 - for i in range(startline, endline+1): - del self.data[startline] + self.data[startline] = self.data[startline][:startchar] + \ + self.data[endline][endchar:] + startline += 1 + for i in range(startline, endline+1): + del self.data[startline] def compare(self, index1, op, index2): line1, char1 = self._decode(index1) diff --git a/Lib/idlelib/idle_test/test_text.py b/Lib/idlelib/idle_test/test_text.py index 69419c1b0ca..367bf384986 100644 --- a/Lib/idlelib/idle_test/test_text.py +++ b/Lib/idlelib/idle_test/test_text.py @@ -36,7 +36,7 @@ def test_index_data(self): self.assertEqual(index(dex), '1.0') for dex in '1.0 lineend', '1.end', '1.33': - self.assertEqual(index(dex), '1.5') + self.assertEqual(index(dex), '1.5') for dex in 'end', '33.44': self.assertEqual(index(dex), '3.0') @@ -225,4 +225,3 @@ def tearDownClass(cls): if __name__ == '__main__': unittest.main(verbosity=2, exit=False) - From 0fd15f8352a8ae706f786b0affe171fba50bf0ff Mon Sep 17 00:00:00 2001 From: R David Murray Date: Fri, 12 Jul 2013 22:57:12 -0400 Subject: [PATCH 221/480] #18437: fix comment typo. --- Lib/email/charset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/email/charset.py b/Lib/email/charset.py index dddaa76c55d..30a13ff2494 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -183,7 +183,7 @@ class Charset: header encoding. Charset.SHORTEST is not allowed for body_encoding. - output_charset: Some character sets must be converted before the can be + output_charset: Some character sets must be converted before they can be used in email headers or bodies. If the input_charset is one of them, this attribute will contain the name of the charset output will be converted to. Otherwise, it will From bbccb8bba3169ae99ce43149be0db02e54b2b0b0 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 13 Jul 2013 00:02:27 -0400 Subject: [PATCH 222/480] Issue #18365: 2.7 corrections so tests run --- Lib/idlelib/idle_test/mock_tk.py | 2 +- Lib/idlelib/idle_test/test_text.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/idlelib/idle_test/mock_tk.py b/Lib/idlelib/idle_test/mock_tk.py index 279f4bef242..d2ab03801a4 100644 --- a/Lib/idlelib/idle_test/mock_tk.py +++ b/Lib/idlelib/idle_test/mock_tk.py @@ -114,7 +114,7 @@ def _decode(self, index, endflag=0): try: index=index.lower() except AttributeError: - raise TclError('bad text index "%s"' % index) from None + raise TclError('bad text index "%s"' % index) lastline = len(self.data) - 1 # same as number of text lines if index == 'insert': diff --git a/Lib/idlelib/idle_test/test_text.py b/Lib/idlelib/idle_test/test_text.py index 367bf384986..3a0705b943c 100644 --- a/Lib/idlelib/idle_test/test_text.py +++ b/Lib/idlelib/idle_test/test_text.py @@ -1,9 +1,9 @@ # Test mock_tk.Text class against tkinter.Text class by running same tests with both. import unittest -from test.support import requires +from test.test_support import requires from _tkinter import TclError -import tkinter as tk +import Tkinter as tk class TextTest(object): @@ -214,7 +214,7 @@ class TkTextTest(TextTest, unittest.TestCase): @classmethod def setUpClass(cls): requires('gui') - from tkinter import Tk, Text + from Tkinter import Tk, Text cls.Text = Text cls.root = Tk() From 077b77290ffe64fb158deb97dc6e57934c5890d3 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 13 Jul 2013 02:34:35 -0400 Subject: [PATCH 223/480] Issue #18279: Add tests for idlelib/RstripExtension.py. Original patch by Phil Webster. With that available, modify RstripExtension.py to stop deleting null slices, which caused a file to be marked as changed when it was not. --- Lib/idlelib/RstripExtension.py | 20 +++++++----- Lib/idlelib/idle_test/mock_idle.py | 27 +++++++++++++++ Lib/idlelib/idle_test/test_rstrip.py | 49 ++++++++++++++++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 5 +++ 5 files changed, 94 insertions(+), 8 deletions(-) create mode 100644 Lib/idlelib/idle_test/mock_idle.py create mode 100644 Lib/idlelib/idle_test/test_rstrip.py diff --git a/Lib/idlelib/RstripExtension.py b/Lib/idlelib/RstripExtension.py index 19e35d4d481..2ce3c7eafe5 100644 --- a/Lib/idlelib/RstripExtension.py +++ b/Lib/idlelib/RstripExtension.py @@ -1,13 +1,9 @@ 'Provides "Strip trailing whitespace" under the "Format" menu.' -__author__ = "Roger D. Serwy " - class RstripExtension: menudefs = [ - ('format', [None, - ('Strip trailing whitespace', '<>'), - ]),] + ('format', [None, ('Strip trailing whitespace', '<>'), ] ), ] def __init__(self, editwin): self.editwin = editwin @@ -20,10 +16,18 @@ def do_rstrip(self, event=None): undo.undo_block_start() - end_line = int(float(text.index('end'))) + 1 + end_line = int(float(text.index('end'))) for cur in range(1, end_line): - txt = text.get('%i.0' % cur, '%i.0 lineend' % cur) + txt = text.get('%i.0' % cur, '%i.end' % cur) + raw = len(txt) cut = len(txt.rstrip()) - text.delete('%i.%i' % (cur, cut), '%i.0 lineend' % cur) + # Since text.delete() marks file as changed, even if not, + # only call it when needed to actually delete something. + if cut < raw: + text.delete('%i.%i' % (cur, cut), '%i.end' % cur) undo.undo_block_stop() + +if __name__ == "__main__": + import unittest + unittest.main('idlelib.idle_test.test_rstrip', verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/mock_idle.py b/Lib/idlelib/idle_test/mock_idle.py new file mode 100644 index 00000000000..54834de2740 --- /dev/null +++ b/Lib/idlelib/idle_test/mock_idle.py @@ -0,0 +1,27 @@ +'''Mock classes that imitate idlelib modules or classes. + +Attributes and methods will be added as needed for tests. +''' + +from idlelib.idle_test.mock_tk import Text + +class Editor(object): + '''Minimally imitate EditorWindow.EditorWindow class. + ''' + def __init__(self, flist=None, filename=None, key=None, root=None): + self.text = Text() + self.undo = UndoDelegator() + + def get_selection_indices(self): + first = self.text.index('1.0') + last = self.text.index('end') + return first, last + +class UndoDelegator(object): + '''Minimally imitate UndoDelegator,UndoDelegator class. + ''' + # A real undo block is only needed for user interaction. + def undo_block_start(*args): + pass + def undo_block_stop(*args): + pass diff --git a/Lib/idlelib/idle_test/test_rstrip.py b/Lib/idlelib/idle_test/test_rstrip.py new file mode 100644 index 00000000000..1c90b93d216 --- /dev/null +++ b/Lib/idlelib/idle_test/test_rstrip.py @@ -0,0 +1,49 @@ +import unittest +import idlelib.RstripExtension as rs +from idlelib.idle_test.mock_idle import Editor + +class rstripTest(unittest.TestCase): + + def test_rstrip_line(self): + editor = Editor() + text = editor.text + do_rstrip = rs.RstripExtension(editor).do_rstrip + + do_rstrip() + self.assertEqual(text.get('1.0', 'insert'), '') + text.insert('1.0', ' ') + do_rstrip() + self.assertEqual(text.get('1.0', 'insert'), '') + text.insert('1.0', ' \n') + do_rstrip() + self.assertEqual(text.get('1.0', 'insert'), '\n') + + def test_rstrip_multiple(self): + editor = Editor() + # Uncomment following to verify that test passes with real widgets. +## from idlelib.EditorWindow import EditorWindow as Editor +## from tkinter import Tk +## editor = Editor(root=Tk()) + text = editor.text + do_rstrip = rs.RstripExtension(editor).do_rstrip + + original = ( + "Line with an ending tab \n" + "Line ending in 5 spaces \n" + "Linewithnospaces\n" + " indented line\n" + " indented line with trailing space \n" + " ") + stripped = ( + "Line with an ending tab\n" + "Line ending in 5 spaces\n" + "Linewithnospaces\n" + " indented line\n" + " indented line with trailing space\n") + + text.insert('1.0', original) + do_rstrip() + self.assertEqual(text.get('1.0', 'insert'), stripped) + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Misc/ACKS b/Misc/ACKS index 82e49566f07..b265036444a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1083,6 +1083,7 @@ Aaron Watters Henrik Weber Corran Webster Glyn Webster +Phil Webster Stefan Wehr Zack Weinberg Bob Weiner diff --git a/Misc/NEWS b/Misc/NEWS index 9bd57913b4d..bbdd6b7b730 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -91,6 +91,10 @@ Build IDLE ---- +- Issue #18279: Format - Strip trailing whitespace no longer marks a file as + changed when it has not been changed. This fix followed the addition of a + test file originally written by Phil Webster (the issue's main goal). + - Issue #7136: In the Idle File menu, "New Window" is renamed "New File". Patch by Tal Einat, Roget Serwy, and Todd Rovito. @@ -104,6 +108,7 @@ IDLE - Issue #15392: Create a unittest framework for IDLE. Preliminary patch by Rajagopalasarma Jayakrishnan + See Lib/idlelib/idle_test/README.txt for how to run Idle tests. - Issue #14146: Highlight source line while debugging on Windows. From b985546c45b04f84c14686b415b017225e62bd4c Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 13 Jul 2013 04:05:42 -0400 Subject: [PATCH 224/480] Issue #18365: convert buildbot errors to skips. --- Lib/idlelib/idle_test/test_text.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/idlelib/idle_test/test_text.py b/Lib/idlelib/idle_test/test_text.py index 3a0705b943c..904ace76d11 100644 --- a/Lib/idlelib/idle_test/test_text.py +++ b/Lib/idlelib/idle_test/test_text.py @@ -216,7 +216,10 @@ def setUpClass(cls): requires('gui') from Tkinter import Tk, Text cls.Text = Text - cls.root = Tk() + try: + cls.root = Tk() + except TclError as msg: + raise unittest.SkipTest('TclError: %s' % msg) @classmethod def tearDownClass(cls): From e4f0f40e2eb8348e01e7ae7eeaa934a7f44622d0 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sun, 14 Jul 2013 12:42:27 -0700 Subject: [PATCH 225/480] =?UTF-8?q?Issue=20#18452:=20fix=20several=20"occu?= =?UTF-8?q?rrence"=20typos=20(reported=20by=20F=C3=A9vry=20Thibault).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Doc/howto/argparse.rst | 2 +- Doc/library/stdtypes.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 4f64948c9d2..3c2b592990d 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -468,7 +468,7 @@ verbosity argument (check the output of ``python --help``):: print answer We have introduced another action, "count", -to count the number of occurences of a specific optional arguments: +to count the number of occurrences of a specific optional arguments: .. code-block:: sh diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 76345e58f7f..0001453e5d0 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -743,10 +743,10 @@ are sequences of the same type; *n*, *i* and *j* are integers: +------------------+--------------------------------+----------+ | ``max(s)`` | largest item of *s* | | +------------------+--------------------------------+----------+ -| ``s.index(i)`` | index of the first occurence | | +| ``s.index(i)`` | index of the first occurrence | | | | of *i* in *s* | | +------------------+--------------------------------+----------+ -| ``s.count(i)`` | total number of occurences of | | +| ``s.count(i)`` | total number of occurrences of | | | | *i* in *s* | | +------------------+--------------------------------+----------+ From 6602086a3e0e4a46164dfa338774563fc1987f84 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 15 Jul 2013 18:35:14 +0200 Subject: [PATCH 226/480] Move entry from #18427 to the right section in the NEWS file --- Misc/NEWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index bbdd6b7b730..e074d7fc08c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,14 +21,14 @@ Core and Builtins - Issue #18019: Fix crash in the repr of dictionaries containing their own views. +- Issue #18427: str.replace could crash the interpreter with huge strings. + Library ------- - Issue #18101: Tcl.split() now process Unicode strings nested in a tuple as it do with byte strings. -- Issue #18427: str.replace could crash the interpreter with huge strings. - - Issue #18347: ElementTree's html serializer now preserves the case of closing tags. From 4b3a5fc15d130d58b6d3082ae4043ee91c2aff53 Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Mon, 15 Jul 2013 18:37:48 +0100 Subject: [PATCH 227/480] Issue #18455: multiprocessing should not retry connect() with same socket. --- Lib/multiprocessing/connection.py | 7 ++++--- Misc/NEWS | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index a0517301510..1a29c36f639 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -294,15 +294,16 @@ def SocketClient(address): ''' Return a connection object connected to the socket given by `address` ''' - family = address_type(address) - s = socket.socket( getattr(socket, family) ) - s.setblocking(True) + family = getattr(socket, address_type(address)) t = _init_timeout() while 1: + s = socket.socket(family) + s.setblocking(True) try: s.connect(address) except socket.error, e: + s.close() if e.args[0] != errno.ECONNREFUSED or _check_timeout(t): debug('failed to connect to address %s', address) raise diff --git a/Misc/NEWS b/Misc/NEWS index e074d7fc08c..0b2dadfb81e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -26,6 +26,8 @@ Core and Builtins Library ------- +- Issue #18455: multiprocessing should not retry connect() with same socket. + - Issue #18101: Tcl.split() now process Unicode strings nested in a tuple as it do with byte strings. From d7f86780361666790d1e5b220c8b075788be87f8 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 15 Jul 2013 19:07:41 -0700 Subject: [PATCH 228/480] =?UTF-8?q?Issue=20#18471:=20Fix=20typo=20in=20hea?= =?UTF-8?q?pq=20documentation=20(reported=20by=20Fran=C3=A7ois=20Pinard).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Doc/library/heapq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index f0723b7b275..e8acd6c2e0a 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -260,7 +260,7 @@ A nice feature of this sort is that you can efficiently insert new items while the sort is going on, provided that the inserted items are not "better" than the last 0'th element you extracted. This is especially useful in simulation contexts, where the tree holds all incoming events, and the "win" condition -means the smallest scheduled time. When an event schedule other events for +means the smallest scheduled time. When an event schedules other events for execution, they are scheduled into the future, so they can easily go into the heap. So, a heap is a good structure for implementing schedulers (this is what I used for my MIDI sequencer :-). From ca8bdcf20fc9deee6fdcad86e955df72511d730e Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 16 Jul 2013 01:59:30 -0700 Subject: [PATCH 229/480] Move the leftlink to the end of the block structure. The current pattern of memory access will update both the leftlink and rightlink at the same time, so they should be positioned side-by-side for better cache locality. Keeping the leftlink at the front of the structure would make sense only if the paired updates were eliminated by backporting changesets 49a9c734304d, 3555cc0ca35b, ae9ee46bd471, and 744dd749e25b. However, that isn't likely to happen, so we're better off with the leftlink at the end of the structure. --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index f175fcaa0ea..b0fa23dfa18 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -47,9 +47,9 @@ */ typedef struct BLOCK { - struct BLOCK *leftlink; PyObject *data[BLOCKLEN]; struct BLOCK *rightlink; + struct BLOCK *leftlink; } block; #define MAXFREEBLOCKS 10 From 7655d2dc7cb688302abb04071f8da1675a4fd73c Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 16 Jul 2013 02:34:19 -0700 Subject: [PATCH 230/480] Backport c952f3d122ae: Tweak the deque struct by moving the least used fields (maxlen and weakref) to the end. --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index b0fa23dfa18..dc31cbc9b0e 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -99,8 +99,8 @@ typedef struct { Py_ssize_t leftindex; /* in range(BLOCKLEN) */ Py_ssize_t rightindex; /* in range(BLOCKLEN) */ Py_ssize_t len; - Py_ssize_t maxlen; long state; /* incremented whenever the indices move */ + Py_ssize_t maxlen; PyObject *weakreflist; /* List of weak references */ } dequeobject; From 4d9556fb12679b312ed2acbe1fdc987651998784 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 16 Jul 2013 22:11:28 +0300 Subject: [PATCH 231/480] Issue #18448: Fix a typo in Demo/newmetaclasses/Eiffel.py. --- Demo/newmetaclasses/Eiffel.py | 2 +- Misc/NEWS | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Demo/newmetaclasses/Eiffel.py b/Demo/newmetaclasses/Eiffel.py index 730a85da5f7..f3f116fe155 100644 --- a/Demo/newmetaclasses/Eiffel.py +++ b/Demo/newmetaclasses/Eiffel.py @@ -29,7 +29,7 @@ def convert_methods(cls, dict): pre = dict.get("%s_pre" % m) post = dict.get("%s_post" % m) if pre or post: - dict[k] = cls.make_eiffel_method(dict[m], pre, post) + dict[m] = cls.make_eiffel_method(dict[m], pre, post) class EiffelMetaClass1(EiffelBaseMetaClass): # an implementation of the "eiffel" meta class that uses nested functions diff --git a/Misc/NEWS b/Misc/NEWS index 0b2dadfb81e..5283da69c17 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -77,6 +77,8 @@ Library Tools/Demos ----------- +- Issue #18448: Fix a typo in Demo/newmetaclasses/Eiffel.py. + - Issue #12990: The "Python Launcher" on OSX could not launch python scripts that have paths that include wide characters. From 4be385a9ec00507c30a29f88fe5aa7ca153a1385 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 20 Jul 2013 10:56:58 -0700 Subject: [PATCH 232/480] Issue #18513: Add workaround for OS X 10.8 cexp bug that leads to wrong cmath.rect(0.0,-0.0) results. --- Misc/NEWS | 3 +++ Modules/cmathmodule.c | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 5283da69c17..87a4ec5ca24 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,6 +28,9 @@ Library - Issue #18455: multiprocessing should not retry connect() with same socket. +- Issue #18513: Fix behaviour of cmath.rect w.r.t. signed zeros on OS X 10.8 + + gcc. + - Issue #18101: Tcl.split() now process Unicode strings nested in a tuple as it do with byte strings. diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 0dc6bdb3968..57206785e71 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -1006,6 +1006,13 @@ cmath_rect(PyObject *self, PyObject *args) else errno = 0; } + else if (phi == 0.0) { + /* Workaround for buggy results with phi=-0.0 on OS X 10.8. See + bugs.python.org/issue18513. */ + z.real = r; + z.imag = r * phi; + errno = 0; + } else { z.real = r * cos(phi); z.imag = r * sin(phi); From 3b31dec136618829388c3089112e74cf51519d3d Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 20 Jul 2013 14:38:24 -0700 Subject: [PATCH 233/480] Issue #17532: Prevent exception when changing key sets if Options menu is empty. --- Lib/idlelib/EditorWindow.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/idlelib/EditorWindow.py b/Lib/idlelib/EditorWindow.py index 14c76de87ff..5de53a99d94 100644 --- a/Lib/idlelib/EditorWindow.py +++ b/Lib/idlelib/EditorWindow.py @@ -833,7 +833,11 @@ def ApplyKeybindings(self): menuEventDict[menu[0]][prepstr(item[0])[1]] = item[1] for menubarItem in self.menudict.keys(): menu = self.menudict[menubarItem] - end = menu.index(END) + 1 + end = menu.index(END) + if end is None: + # Skip empty menus + continue + end += 1 for index in range(0, end): if menu.type(index) == 'command': accel = menu.entrycget(index, 'accelerator') From 871fadb715c1c82f17a72d8429f17720bd679a0e Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 21 Jul 2013 20:13:24 -0400 Subject: [PATCH 234/480] Issue #18441: Make test.support.requires('gui') skip when it should. (Consolidating this check and various checks in tkinter files and moving them to test.support and test.regrtest will be another issue.) --- Lib/idlelib/idle_test/test_text.py | 5 +---- Lib/test/test_idle.py | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Lib/idlelib/idle_test/test_text.py b/Lib/idlelib/idle_test/test_text.py index 904ace76d11..3a0705b943c 100644 --- a/Lib/idlelib/idle_test/test_text.py +++ b/Lib/idlelib/idle_test/test_text.py @@ -216,10 +216,7 @@ def setUpClass(cls): requires('gui') from Tkinter import Tk, Text cls.Text = Text - try: - cls.root = Tk() - except TclError as msg: - raise unittest.SkipTest('TclError: %s' % msg) + cls.root = Tk() @classmethod def tearDownClass(cls): diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 4262aa3f679..540f7d20643 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -1,9 +1,21 @@ -# Skip test if _tkinter or _thread wasn't built or idlelib was deleted. -from test.test_support import import_module -import_module('Tkinter') -import_module('threading') # imported by PyShell, imports _thread +# Skip test if _thread or _tkinter wasn't built or idlelib was deleted. +from test.test_support import import_module, use_resources +import_module('threading') # imported by idlelib.PyShell, imports _thread +tk = import_module('Tkinter') idletest = import_module('idlelib.idle_test') +# If buildbot improperly sets gui resource (#18365, #18441), remove it +# so requires('gui') tests are skipped while non-gui tests still run. +if use_resources and 'gui' in use_resources: + try: + root = tk.Tk() + root.destroy() + except TclError: + while True: + use_resources.delete('gui') + if 'gui' not in use_resources: + break + # Without test_main present, regrtest.runtest_inner (line1219) calls # unittest.TestLoader().loadTestsFromModule(this_module) which calls # load_tests() if it finds it. (Unittest.main does the same.) From b368e8e65e9eb2b218cbd07717f44dafa4f17c0a Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 21 Jul 2013 20:57:44 -0400 Subject: [PATCH 235/480] Issue #18439: Make patchcheck work on Windows for ACKS, NEWS. --- Misc/NEWS | 2 ++ Tools/scripts/patchcheck.py | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 87a4ec5ca24..72e44e25121 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -137,6 +137,8 @@ Documentation - Issue #17844: Refactor a documentation of Python specific encodings. Add links to encoders and decoders for binary-to-binary codecs. +- Issue #18439: Make patchcheck work on Windows for ACKS, NEWS. + What's New in Python 2.7.5? =========================== diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index fe2e2913f1f..418dd269daf 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -144,13 +144,13 @@ def docs_modified(file_paths): @status("Misc/ACKS updated", modal=True) def credit_given(file_paths): """Check if Misc/ACKS has been changed.""" - return 'Misc/ACKS' in file_paths + return os.path.join('Misc', 'ACKS') in file_paths @status("Misc/NEWS updated", modal=True) def reported_news(file_paths): """Check if Misc/NEWS has been changed.""" - return 'Misc/NEWS' in file_paths + return os.path.join('Misc', 'NEWS') in file_paths def main(): @@ -158,7 +158,8 @@ def main(): python_files = [fn for fn in file_paths if fn.endswith('.py')] c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] doc_files = [fn for fn in file_paths if fn.startswith('Doc')] - special_files = {'Misc/ACKS', 'Misc/NEWS'} & set(file_paths) + misc_files = {os.path.join('Misc', 'ACKS'), os.path.join('Misc', 'NEWS')}\ + & set(file_paths) # PEP 8 whitespace rules enforcement. normalize_whitespace(python_files) # C rules enforcement. @@ -168,9 +169,9 @@ def main(): # Docs updated. docs_modified(doc_files) # Misc/ACKS changed. - credit_given(special_files) + credit_given(misc_files) # Misc/NEWS changed. - reported_news(special_files) + reported_news(misc_files) # Test suite run and passed. if python_files or c_files: From 1dd39fe4030f4551b0ea8bf4709814a9b18bd9af Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 21 Jul 2013 21:08:29 -0400 Subject: [PATCH 236/480] Issue #18439: Move news entry misplaced by graft merge. --- Misc/NEWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 72e44e25121..e02d23d91f6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -80,6 +80,8 @@ Library Tools/Demos ----------- +- Issue #18439: Make patchcheck work on Windows for ACKS, NEWS. + - Issue #18448: Fix a typo in Demo/newmetaclasses/Eiffel.py. - Issue #12990: The "Python Launcher" on OSX could not launch python scripts @@ -137,8 +139,6 @@ Documentation - Issue #17844: Refactor a documentation of Python specific encodings. Add links to encoders and decoders for binary-to-binary codecs. -- Issue #18439: Make patchcheck work on Windows for ACKS, NEWS. - What's New in Python 2.7.5? =========================== From 5bf32f18354454f4ed84e35a821c25bdc343a54b Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 21 Jul 2013 21:24:45 -0400 Subject: [PATCH 237/480] Issue #18441: fix buildbot name-error for TclError. --- Lib/test/test_idle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 540f7d20643..79d18a809ba 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -10,7 +10,7 @@ try: root = tk.Tk() root.destroy() - except TclError: + except tk.TclError: while True: use_resources.delete('gui') if 'gui' not in use_resources: From 58c6e83fc7c25be8577257358eeabb00624096f5 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 21 Jul 2013 21:57:02 -0400 Subject: [PATCH 238/480] Backed out changeset: 9f922270a929 so can backout larger patch --- Lib/test/test_idle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 79d18a809ba..540f7d20643 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -10,7 +10,7 @@ try: root = tk.Tk() root.destroy() - except tk.TclError: + except TclError: while True: use_resources.delete('gui') if 'gui' not in use_resources: From a8841f7562461566e98e7a0256e0218cc60538df Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 21 Jul 2013 21:57:52 -0400 Subject: [PATCH 239/480] Backed out changeset: 23b0164b9c82 #18441 not working --- Lib/idlelib/idle_test/test_text.py | 5 ++++- Lib/test/test_idle.py | 20 ++++---------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/Lib/idlelib/idle_test/test_text.py b/Lib/idlelib/idle_test/test_text.py index 3a0705b943c..904ace76d11 100644 --- a/Lib/idlelib/idle_test/test_text.py +++ b/Lib/idlelib/idle_test/test_text.py @@ -216,7 +216,10 @@ def setUpClass(cls): requires('gui') from Tkinter import Tk, Text cls.Text = Text - cls.root = Tk() + try: + cls.root = Tk() + except TclError as msg: + raise unittest.SkipTest('TclError: %s' % msg) @classmethod def tearDownClass(cls): diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 540f7d20643..4262aa3f679 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -1,21 +1,9 @@ -# Skip test if _thread or _tkinter wasn't built or idlelib was deleted. -from test.test_support import import_module, use_resources -import_module('threading') # imported by idlelib.PyShell, imports _thread -tk = import_module('Tkinter') +# Skip test if _tkinter or _thread wasn't built or idlelib was deleted. +from test.test_support import import_module +import_module('Tkinter') +import_module('threading') # imported by PyShell, imports _thread idletest = import_module('idlelib.idle_test') -# If buildbot improperly sets gui resource (#18365, #18441), remove it -# so requires('gui') tests are skipped while non-gui tests still run. -if use_resources and 'gui' in use_resources: - try: - root = tk.Tk() - root.destroy() - except TclError: - while True: - use_resources.delete('gui') - if 'gui' not in use_resources: - break - # Without test_main present, regrtest.runtest_inner (line1219) calls # unittest.TestLoader().loadTestsFromModule(this_module) which calls # load_tests() if it finds it. (Unittest.main does the same.) From 48d98c8aafe528f2686ea38ce2e02ec70d6fc936 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Wed, 24 Jul 2013 17:47:52 +0100 Subject: [PATCH 240/480] Issue #18541: simplified LoggerAdapter example. --- Doc/howto/logging-cookbook.rst | 76 +++++++++------------------------- 1 file changed, 19 insertions(+), 57 deletions(-) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 422bf624c74..00d0e809143 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -460,69 +460,31 @@ customized strings with your :class:`Formatter` instances which know about the keys of the dict-like object. If you need a different method, e.g. if you want to prepend or append the contextual information to the message string, you just need to subclass :class:`LoggerAdapter` and override :meth:`process` -to do what you need. Here's an example script which uses this class, which -also illustrates what dict-like behaviour is needed from an arbitrary -'dict-like' object for use in the constructor:: +to do what you need. Here is a simple example:: - import logging + class CustomAdapter(logging.LoggerAdapter): + """ + This example adapter expects the passed in dict-like object to have a + 'connid' key, whose value in brackets is prepended to the log message. + """ + def process(self, msg, kwargs): + return '[%s] %s' % (self.extra['connid'], msg), kwargs - class ConnInfo: - """ - An example class which shows how an arbitrary class can be used as - the 'extra' context information repository passed to a LoggerAdapter. - """ +which you can use like this:: - def __getitem__(self, name): - """ - To allow this instance to look like a dict. - """ - from random import choice - if name == 'ip': - result = choice(['127.0.0.1', '192.168.0.1']) - elif name == 'user': - result = choice(['jim', 'fred', 'sheila']) - else: - result = self.__dict__.get(name, '?') - return result + logger = logging.getLogger(__name__) + adapter = CustomAdapter(logger, {'connid': some_conn_id}) - def __iter__(self): - """ - To allow iteration over keys, which will be merged into - the LogRecord dict before formatting and output. - """ - keys = ['ip', 'user'] - keys.extend(self.__dict__.keys()) - return keys.__iter__() +Then any events that you log to the adapter will have the value of +``some_conn_id`` prepended to the log messages. - if __name__ == '__main__': - from random import choice - levels = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL) - a1 = logging.LoggerAdapter(logging.getLogger('a.b.c'), - { 'ip' : '123.231.231.123', 'user' : 'sheila' }) - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)-15s %(name)-5s %(levelname)-8s IP: %(ip)-15s User: %(user)-8s %(message)s') - a1.debug('A debug message') - a1.info('An info message with %s', 'some parameters') - a2 = logging.LoggerAdapter(logging.getLogger('d.e.f'), ConnInfo()) - for x in range(10): - lvl = choice(levels) - lvlname = logging.getLevelName(lvl) - a2.log(lvl, 'A message at %s level with %d %s', lvlname, 2, 'parameters') +Using objects other than dicts to pass contextual information +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -When this script is run, the output should look something like this:: - - 2008-01-18 14:49:54,023 a.b.c DEBUG IP: 123.231.231.123 User: sheila A debug message - 2008-01-18 14:49:54,023 a.b.c INFO IP: 123.231.231.123 User: sheila An info message with some parameters - 2008-01-18 14:49:54,023 d.e.f CRITICAL IP: 192.168.0.1 User: jim A message at CRITICAL level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f INFO IP: 192.168.0.1 User: jim A message at INFO level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f WARNING IP: 192.168.0.1 User: sheila A message at WARNING level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f ERROR IP: 127.0.0.1 User: fred A message at ERROR level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f ERROR IP: 127.0.0.1 User: sheila A message at ERROR level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f WARNING IP: 192.168.0.1 User: sheila A message at WARNING level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f WARNING IP: 192.168.0.1 User: jim A message at WARNING level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f INFO IP: 192.168.0.1 User: fred A message at INFO level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f WARNING IP: 192.168.0.1 User: sheila A message at WARNING level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f WARNING IP: 127.0.0.1 User: jim A message at WARNING level with 2 parameters +You don't need to pass an actual dict to a :class:`LoggerAdapter` - you could +pass an instance of a class which implements ``__getitem__`` and ``__iter__`` so +that it looks like a dict to logging. This would be useful if you want to +generate values dynamically (whereas the values in a dict would be constant). .. _filters-contextual: From 2f3c7693b10a65173a9e4bfd55f671e07a83ab2d Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Thu, 25 Jul 2013 05:21:00 +0200 Subject: [PATCH 241/480] #14853: remove test that was making too many assumptions about stdin. Patch by Elena Oat. --- Lib/test/test_file.py | 10 ---------- Misc/ACKS | 1 + 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index dffa4b56eb6..ac2aafa1ce4 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -154,16 +154,6 @@ def testModeStrings(self): f.close() self.fail('%r is an invalid file mode' % mode) - def testStdin(self): - # This causes the interpreter to exit on OSF1 v5.1. - if sys.platform != 'osf1V5': - self.assertRaises((IOError, ValueError), sys.stdin.seek, -1) - else: - print(( - ' Skipping sys.stdin.seek(-1), it may crash the interpreter.' - ' Test manually.'), file=sys.__stdout__) - self.assertRaises((IOError, ValueError), sys.stdin.truncate) - def testBadModeArgument(self): # verify that we get a sensible error message for bad mode argument bad_mode = "qwerty" diff --git a/Misc/ACKS b/Misc/ACKS index b265036444a..cf553b70e6f 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -744,6 +744,7 @@ Nigel O'Brian Kevin O'Connor Tim O'Malley Zooko O'Whielacronx +Elena Oat Pascal Oberndoerfer Jeffrey Ollie Adam Olsen From f0305f05b85c684a8833a2fb42ce2e3703654d45 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Thu, 25 Jul 2013 13:33:35 -0400 Subject: [PATCH 242/480] #15130: remove repeat of abstract paragraph from socket howto body. Patch by Tshepang Lekhonkhobe. --- Doc/howto/sockets.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index 6338d3b846e..e0083edc51c 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -19,12 +19,6 @@ Sockets ======= -Sockets are used nearly everywhere, but are one of the most severely -misunderstood technologies around. This is a 10,000 foot overview of sockets. -It's not really a tutorial - you'll still have work to do in getting things -working. It doesn't cover the fine points (and there are a lot of them), but I -hope it will give you enough background to begin using them decently. - I'm only going to talk about INET sockets, but they account for at least 99% of the sockets in use. And I'll only talk about STREAM sockets - unless you really know what you're doing (in which case this HOWTO isn't for you!), you'll get From ab2cd61435fecae41baa31106061b7b1897bc0a1 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 26 Jul 2013 18:21:32 -0400 Subject: [PATCH 243/480] Issue #18539: Calltips now work for float default arguments. --- Lib/idlelib/CallTips.py | 2 +- Lib/idlelib/idle_test/test_calltips.py | 6 ++++++ Misc/NEWS | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/idlelib/CallTips.py b/Lib/idlelib/CallTips.py index 0fdef11c699..c29f89bc0dd 100644 --- a/Lib/idlelib/CallTips.py +++ b/Lib/idlelib/CallTips.py @@ -163,7 +163,7 @@ def get_arg_text(ob): if fob.func_code.co_flags & 0x8: items.append("***") arg_text = ", ".join(items) - arg_text = "(%s)" % re.sub("\.\d+", "", arg_text) + arg_text = "(%s)" % re.sub("(?", arg_text) # See if we can use the docstring doc = getattr(ob, "__doc__", "") if doc: diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py index 1733d392b35..e0f1665fa71 100644 --- a/Lib/idlelib/idle_test/test_calltips.py +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -10,5 +10,11 @@ def test_bad_entity(self): def test_good_entity(self): self.assertIs(CTi.get_entity('int'), int) +class Py2Test(unittest.TestCase): + def test_paramtuple_float(self): + # 18539: (a,b) becomes '.0' in code object; change that but not float + def f((a,b), c=0.0): pass + self.assertEqual(ct.get_arg_text(f), '(, c=0.0)') + if __name__ == '__main__': unittest.main(verbosity=2, exit=False) diff --git a/Misc/NEWS b/Misc/NEWS index e02d23d91f6..25193802888 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -104,6 +104,8 @@ IDLE changed when it has not been changed. This fix followed the addition of a test file originally written by Phil Webster (the issue's main goal). +- Issue #18539: Calltips now work for float default arguments. + - Issue #7136: In the Idle File menu, "New Window" is renamed "New File". Patch by Tal Einat, Roget Serwy, and Todd Rovito. From 292b124ac2b660b3983afad2f204ce03112d579c Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 27 Jul 2013 15:54:05 -0400 Subject: [PATCH 244/480] Remove repetition of string escapes note 1 at end of note 2. --- Doc/reference/lexical_analysis.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index ea92b8ed87a..2f31c853f50 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -529,8 +529,7 @@ Notes: (2) Any Unicode character can be encoded this way, but characters outside the Basic Multilingual Plane (BMP) will be encoded using a surrogate pair if Python is - compiled to use 16-bit code units (the default). Individual code units which - form parts of a surrogate pair can be encoded using this escape sequence. + compiled to use 16-bit code units (the default). (3) As in Standard C, up to three octal digits are accepted. From 8a0206708f8b1415208c1766bfc61c12aba432a6 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 21 Jul 2013 20:13:24 -0400 Subject: [PATCH 245/480] Issue #18441: Make test.support.requires('gui') skip when it should. (Consolidating this check and various checks in tkinter files and moving them to test.support and test.regrtest will be another issue.) --- Lib/idlelib/idle_test/test_text.py | 5 +---- Lib/test/test_idle.py | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Lib/idlelib/idle_test/test_text.py b/Lib/idlelib/idle_test/test_text.py index 904ace76d11..3a0705b943c 100644 --- a/Lib/idlelib/idle_test/test_text.py +++ b/Lib/idlelib/idle_test/test_text.py @@ -216,10 +216,7 @@ def setUpClass(cls): requires('gui') from Tkinter import Tk, Text cls.Text = Text - try: - cls.root = Tk() - except TclError as msg: - raise unittest.SkipTest('TclError: %s' % msg) + cls.root = Tk() @classmethod def tearDownClass(cls): diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 4262aa3f679..540f7d20643 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -1,9 +1,21 @@ -# Skip test if _tkinter or _thread wasn't built or idlelib was deleted. -from test.test_support import import_module -import_module('Tkinter') -import_module('threading') # imported by PyShell, imports _thread +# Skip test if _thread or _tkinter wasn't built or idlelib was deleted. +from test.test_support import import_module, use_resources +import_module('threading') # imported by idlelib.PyShell, imports _thread +tk = import_module('Tkinter') idletest = import_module('idlelib.idle_test') +# If buildbot improperly sets gui resource (#18365, #18441), remove it +# so requires('gui') tests are skipped while non-gui tests still run. +if use_resources and 'gui' in use_resources: + try: + root = tk.Tk() + root.destroy() + except TclError: + while True: + use_resources.delete('gui') + if 'gui' not in use_resources: + break + # Without test_main present, regrtest.runtest_inner (line1219) calls # unittest.TestLoader().loadTestsFromModule(this_module) which calls # load_tests() if it finds it. (Unittest.main does the same.) From a52b5827ecbff5ac5a4d4f7a80dd81d71f1cc0dd Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 27 Jul 2013 19:07:07 -0400 Subject: [PATCH 246/480] Issue #18441: Correct previous patch, which hg committed before I wanted it to. --- Lib/test/test_idle.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 540f7d20643..fc05be1870e 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -1,4 +1,5 @@ # Skip test if _thread or _tkinter wasn't built or idlelib was deleted. +from test import test_support as support from test.test_support import import_module, use_resources import_module('threading') # imported by idlelib.PyShell, imports _thread tk = import_module('Tkinter') @@ -10,11 +11,9 @@ try: root = tk.Tk() root.destroy() - except TclError: - while True: - use_resources.delete('gui') - if 'gui' not in use_resources: - break + except tk.TclError: + while 'gui' in use_resources: + use_resources.remove('gui') # Without test_main present, regrtest.runtest_inner (line1219) calls # unittest.TestLoader().loadTestsFromModule(this_module) which calls @@ -25,7 +24,6 @@ # Until unittest supports resources, we emulate regrtest's -ugui # so loaded tests run the same as if textually present here. # If any Idle test ever needs another resource, add it to the list. - from test import support support.use_resources = ['gui'] # use_resources is initially None import unittest unittest.main(verbosity=2, exit=False) From 5de147cfe5b95fce43202ccb8b348f117826242e Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 27 Jul 2013 20:28:53 -0400 Subject: [PATCH 247/480] Update int division to quiet 2to3 warning. --- Lib/lib-tk/test/test_ttk/test_widgets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/lib-tk/test/test_ttk/test_widgets.py b/Lib/lib-tk/test/test_ttk/test_widgets.py index 9d06a75610e..2fcb3fa7816 100644 --- a/Lib/lib-tk/test/test_ttk/test_widgets.py +++ b/Lib/lib-tk/test/test_ttk/test_widgets.py @@ -26,8 +26,8 @@ def tearDown(self): def test_identify(self): self.widget.update_idletasks() self.assertEqual(self.widget.identify( - int(self.widget.winfo_width() / 2), - int(self.widget.winfo_height() / 2) + self.widget.winfo_width() // 2, + self.widget.winfo_height() // 2 ), "label") self.assertEqual(self.widget.identify(-1, -1), "") From 71f69a4b53a676e13e95ba856773bec532b65ae7 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 27 Jul 2013 22:06:03 -0400 Subject: [PATCH 248/480] Issue #18441: add Mac (darwin) gui check. This is not needed today, but has been in the past and might be needed again in the future --- Lib/test/test_idle.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index fc05be1870e..77f6b1de519 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -1,14 +1,24 @@ -# Skip test if _thread or _tkinter wasn't built or idlelib was deleted. +import unittest from test import test_support as support from test.test_support import import_module, use_resources + +# Skip test if _thread or _tkinter wasn't built or idlelib was deleted. import_module('threading') # imported by idlelib.PyShell, imports _thread -tk = import_module('Tkinter') +tk = import_module('Tkinter') # imports _tkinter idletest = import_module('idlelib.idle_test') # If buildbot improperly sets gui resource (#18365, #18441), remove it # so requires('gui') tests are skipped while non-gui tests still run. if use_resources and 'gui' in use_resources: try: + import sys + if sys.platform == 'darwin': + from lib-tk.test.runtktests import check_tk_availability + # tkinter.test.suppport in 3.x + try: + check_tk_availability() + except unittest.SkipTest: + raise tk.TclError root = tk.Tk() root.destroy() except tk.TclError: @@ -25,5 +35,4 @@ # so loaded tests run the same as if textually present here. # If any Idle test ever needs another resource, add it to the list. support.use_resources = ['gui'] # use_resources is initially None - import unittest unittest.main(verbosity=2, exit=False) From 92fa6e7ba4ccca456b5befbc647657600f0edc10 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 27 Jul 2013 22:27:25 -0400 Subject: [PATCH 249/480] Issue #18441: Comment out code that will not compile because the standard library has a package, lib-tk, that cannot be imported by normal means. Lib/test/test_tk, etc, have special code to access this package. I will not bother with it unless the darwin check is needed before it gets moved to test.(test-)support.py. --- Lib/test/test_idle.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 77f6b1de519..177e3eb9739 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -12,13 +12,14 @@ if use_resources and 'gui' in use_resources: try: import sys - if sys.platform == 'darwin': - from lib-tk.test.runtktests import check_tk_availability - # tkinter.test.suppport in 3.x - try: - check_tk_availability() - except unittest.SkipTest: - raise tk.TclError +## if sys.platform == 'darwin': +## from lib-tk.test.runtktests import check_tk_availability +# see test/test_tk.py or test_ttk_guionly for how to import the above +## # tkinter.test.suppport in 3.x +## try: +## check_tk_availability() +## except unittest.SkipTest: +## raise tk.TclError root = tk.Tk() root.destroy() except tk.TclError: From 875bc24284adb14fdcb52e3681eadab0291d9db0 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 27 Jul 2013 22:29:09 -0400 Subject: [PATCH 250/480] Issue #18441: whitespace --- Lib/test/test_idle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 177e3eb9739..b94672c44ce 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -18,7 +18,7 @@ ## # tkinter.test.suppport in 3.x ## try: ## check_tk_availability() -## except unittest.SkipTest: +## except unittest.SkipTest: ## raise tk.TclError root = tk.Tk() root.destroy() From 2b3cfb372fc7936a584eaa1c8bca0614c97a9bce Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 28 Jul 2013 00:01:00 -0400 Subject: [PATCH 251/480] Issue #18441: Move commented out code to issue message. --- Lib/test/test_idle.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index b94672c44ce..8552e6fbe44 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -9,17 +9,9 @@ # If buildbot improperly sets gui resource (#18365, #18441), remove it # so requires('gui') tests are skipped while non-gui tests still run. +# If there is a problem with Macs, see #18441, msg 193805 if use_resources and 'gui' in use_resources: try: - import sys -## if sys.platform == 'darwin': -## from lib-tk.test.runtktests import check_tk_availability -# see test/test_tk.py or test_ttk_guionly for how to import the above -## # tkinter.test.suppport in 3.x -## try: -## check_tk_availability() -## except unittest.SkipTest: -## raise tk.TclError root = tk.Tk() root.destroy() except tk.TclError: From f8aa812ce39fe30e7cbb1ac26027577d0dae6411 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 28 Jul 2013 02:34:42 -0700 Subject: [PATCH 252/480] Restore the data block size to 62. The former block size traded away good fit within cache lines in order to gain faster division in deque_item(). However, compilers are getting smarter and can now replace the slow division operation with a fast integer multiply and right shift. Accordingly, it makes sense to go back to a size that lets blocks neatly fill entire cache-lines. GCC-4.8 and CLANG 4.0 both compute "x // 62" with something roughly equivalent to "x * 9520900167075897609 >> 69". --- Lib/test/test_deque.py | 2 +- Modules/_collectionsmodule.c | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 98b203edb60..595a0c4a35c 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -522,7 +522,7 @@ class C(object): @test_support.cpython_only def test_sizeof(self): - BLOCKLEN = 64 + BLOCKLEN = 62 basesize = test_support.calcobjsize('2P4PlP') blocksize = struct.calcsize('2P%dP' % BLOCKLEN) self.assertEqual(object.__sizeof__(deque()), basesize) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index dc31cbc9b0e..26d878344e1 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -10,11 +10,14 @@ /* The block length may be set to any number over 1. Larger numbers * reduce the number of calls to the memory allocator, give faster * indexing and rotation, and reduce the link::data overhead ratio. - * Ideally, the block length should be a power-of-two for faster - * division/modulo computations during indexing. + * + * Ideally, the block length will be set to two less than some + * multiple of the cache-line length (so that the full block + * including the leftlink and rightlink will fit neatly into + * cache lines). */ -#define BLOCKLEN 64 +#define BLOCKLEN 62 #define CENTER ((BLOCKLEN - 1) / 2) /* A `dequeobject` is composed of a doubly-linked list of `block` nodes. From 977b1dd5c428a8d3459dced001c2151f6c483876 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 28 Jul 2013 16:39:44 -0400 Subject: [PATCH 253/480] Update gui section of idle test README. --- Lib/idlelib/idle_test/README.txt | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Lib/idlelib/idle_test/README.txt b/Lib/idlelib/idle_test/README.txt index b498e7ce1f8..a8d4dcb9922 100644 --- a/Lib/idlelib/idle_test/README.txt +++ b/Lib/idlelib/idle_test/README.txt @@ -40,13 +40,20 @@ master object either directly or indirectly by instantiating a tkinter or idle class. For the benefit of buildbot machines that do not have a graphics screen, gui tests must be 'guarded' by "requires('gui')" in a setUp function or method. This will typically be setUpClass. + +All gui objects must be destroyed by the end of the test, perhaps in a tearDown +function. Creating the Tk root directly in a setUp allows a reference to be saved +so it can be properly destroyed in the corresponding tearDown. --- @classmethod def setUpClass(cls): requires('gui') + cls.root = tk.Tk() + + @classmethod + def tearDownClass(cls): + cls.root.destroy() --- -All gui objects must be destroyed by the end of the test, perhaps in a tearDown -function. Support.requires('gui') returns true if it is either called in a main module (which never happens on buildbots) or if use_resources contains 'gui'. @@ -56,8 +63,9 @@ template above. Since non-gui tests always run, but gui tests only sometimes, tests of non-gui operations should best avoid needing a gui. Methods that make incidental use of -tkinter variables and messageboxes can do this by using the mock classes in -idle_test/mock_tk.py. +tkinter (tk) variables and messageboxes can do this by using the mock classes in +idle_test/mock_tk.py. There is also a mock text that will handle some uses of the +tk Text widget. 3. Running Tests From c32e24db028a91e36bf246b02ef999b674c24e21 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Tue, 30 Jul 2013 01:37:36 -0400 Subject: [PATCH 254/480] Make all idle test case names end with 'Test'. --- Lib/idlelib/idle_test/test_calltips.py | 2 +- Lib/idlelib/idle_test/test_config_name.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py index e0f1665fa71..79884f21a6f 100644 --- a/Lib/idlelib/idle_test/test_calltips.py +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -2,7 +2,7 @@ import idlelib.CallTips as ct CTi = ct.CallTips() -class Test_get_entity(unittest.TestCase): +class Get_entityTest(unittest.TestCase): # In 3.x, get_entity changed from 'instance method' to module function # since 'self' not used. Use dummy instance until change 2.7 also. def test_bad_entity(self): diff --git a/Lib/idlelib/idle_test/test_config_name.py b/Lib/idlelib/idle_test/test_config_name.py index a38c896e58e..4403f87fd3b 100644 --- a/Lib/idlelib/idle_test/test_config_name.py +++ b/Lib/idlelib/idle_test/test_config_name.py @@ -22,7 +22,7 @@ def destroy(self): orig_mbox = name_dialog_module.tkMessageBox showerror = Mbox.showerror -class TestConfigName(unittest.TestCase): +class ConfigNameTest(unittest.TestCase): dialog = Dummy_name_dialog() @classmethod From a21335199e4305a12c44d8614e52459234bca92e Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Wed, 31 Jul 2013 00:14:20 -0700 Subject: [PATCH 255/480] Issue #18071: Extension module builds on OS X could fail with TypeError if Xcode command line tools were not installed. --- Lib/_osx_support.py | 4 ++-- Misc/NEWS | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 79dbdc9ea3d..195561aff30 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -53,7 +53,7 @@ def _find_executable(executable, path=None): def _read_output(commandstring): - """Output from succesful command execution or None""" + """Output from successful command execution or None""" # Similar to os.popen(commandstring, "r").read(), # but without actually using os.popen because that # function is not usable during python bootstrap. @@ -68,7 +68,7 @@ def _read_output(commandstring): with contextlib.closing(fp) as fp: cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) - return fp.read().decode('utf-8').strip() if not os.system(cmd) else None + return fp.read().strip() if not os.system(cmd) else None def _find_build_tool(toolname): diff --git a/Misc/NEWS b/Misc/NEWS index 25193802888..ee5c8f044fd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -74,6 +74,9 @@ Library the default for linking if LDSHARED is not also overriden. This restores Distutils behavior introduced in 2.7.3 and inadvertently dropped in 2.7.4. +- Issue #18071: C extension module builds on OS X could fail with TypeError + if the Xcode command line tools were not installed. + - Issue #18113: Fixed a refcount leak in the curses.panel module's set_userptr() method. Reported by Atsuo Ishimoto. From 7b8cd25ae0eae6c69436ac5dc2b7f2f8987226ae Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 1 Aug 2013 21:17:24 +0200 Subject: [PATCH 256/480] Issue #18589: fix hyperlinking of type slots (tp_*) --- Doc/c-api/allocation.rst | 6 +- Doc/c-api/gcsupport.rst | 28 ++-- Doc/c-api/structures.rst | 4 +- Doc/c-api/typeobj.rst | 280 ++++++++++++++++++------------------- Doc/extending/newtypes.rst | 98 ++++++------- Doc/library/gc.rst | 4 +- Doc/library/stdtypes.rst | 6 +- Doc/whatsnew/2.2.rst | 6 +- 8 files changed, 216 insertions(+), 216 deletions(-) diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index cb43cbfc450..32a414b1aca 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -43,7 +43,7 @@ Allocating Objects on the Heap Allocate a new Python object using the C structure type *TYPE* and the Python type object *type*. Fields not defined by the Python object header are not initialized; the object's reference count will be one. The size of - the memory allocation is determined from the :attr:`tp_basicsize` field of + the memory allocation is determined from the :c:member:`~PyTypeObject.tp_basicsize` field of the type object. @@ -52,7 +52,7 @@ Allocating Objects on the Heap Allocate a new Python object using the C structure type *TYPE* and the Python type object *type*. Fields not defined by the Python object header are not initialized. The allocated memory allows for the *TYPE* structure - plus *size* fields of the size given by the :attr:`tp_itemsize` field of + plus *size* fields of the size given by the :c:member:`~PyTypeObject.tp_itemsize` field of *type*. This is useful for implementing objects like tuples, which are able to determine their size at construction time. Embedding the array of fields into the same allocation decreases the number of allocations, @@ -67,7 +67,7 @@ Allocating Objects on the Heap Releases memory allocated to an object using :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`. This is normally called from the - :attr:`tp_dealloc` handler specified in the object's type. The fields of + :c:member:`~PyTypeObject.tp_dealloc` handler specified in the object's type. The fields of the object should not be accessed after this call as the memory is no longer a valid Python object. diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 2a4fda4f3c2..b0a2d5c08dd 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -15,10 +15,10 @@ collection. .. An example showing the use of these interfaces can be found in "Supporting the .. Cycle Collector (XXX not found: ../ext/example-cycle-support.html)". -To create a container type, the :attr:`tp_flags` field of the type object must +To create a container type, the :c:member:`~PyTypeObject.tp_flags` field of the type object must include the :const:`Py_TPFLAGS_HAVE_GC` and provide an implementation of the -:attr:`tp_traverse` handler. If instances of the type are mutable, a -:attr:`tp_clear` implementation must also be provided. +:c:member:`~PyTypeObject.tp_traverse` handler. If instances of the type are mutable, a +:c:member:`~PyTypeObject.tp_clear` implementation must also be provided. .. data:: Py_TPFLAGS_HAVE_GC @@ -68,7 +68,7 @@ Constructors for container types must conform to two rules: Adds the object *op* to the set of container objects tracked by the collector. The collector can run at unexpected times so objects must be valid while being tracked. This should be called once all the fields - followed by the :attr:`tp_traverse` handler become valid, usually near the + followed by the :c:member:`~PyTypeObject.tp_traverse` handler become valid, usually near the end of the constructor. @@ -97,8 +97,8 @@ rules: Remove the object *op* from the set of container objects tracked by the collector. Note that :c:func:`PyObject_GC_Track` can be called again on this object to add it back to the set of tracked objects. The deallocator - (:attr:`tp_dealloc` handler) should call this for the object before any of - the fields used by the :attr:`tp_traverse` handler become invalid. + (:c:member:`~PyTypeObject.tp_dealloc` handler) should call this for the object before any of + the fields used by the :c:member:`~PyTypeObject.tp_traverse` handler become invalid. .. c:function:: void _PyObject_GC_UNTRACK(PyObject *op) @@ -106,19 +106,19 @@ rules: A macro version of :c:func:`PyObject_GC_UnTrack`. It should not be used for extension modules. -The :attr:`tp_traverse` handler accepts a function parameter of this type: +The :c:member:`~PyTypeObject.tp_traverse` handler accepts a function parameter of this type: .. c:type:: int (*visitproc)(PyObject *object, void *arg) - Type of the visitor function passed to the :attr:`tp_traverse` handler. + Type of the visitor function passed to the :c:member:`~PyTypeObject.tp_traverse` handler. The function should be called with an object to traverse as *object* and - the third parameter to the :attr:`tp_traverse` handler as *arg*. The + the third parameter to the :c:member:`~PyTypeObject.tp_traverse` handler as *arg*. The Python core uses several visitor functions to implement cyclic garbage detection; it's not expected that users will need to write their own visitor functions. -The :attr:`tp_traverse` handler must have the following type: +The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: .. c:type:: int (*traverseproc)(PyObject *self, visitproc visit, void *arg) @@ -130,15 +130,15 @@ The :attr:`tp_traverse` handler must have the following type: object argument. If *visit* returns a non-zero value that value should be returned immediately. -To simplify writing :attr:`tp_traverse` handlers, a :c:func:`Py_VISIT` macro is -provided. In order to use this macro, the :attr:`tp_traverse` implementation +To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is +provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation must name its arguments exactly *visit* and *arg*: .. c:function:: void Py_VISIT(PyObject *o) Call the *visit* callback, with arguments *o* and *arg*. If *visit* returns - a non-zero value, then return it. Using this macro, :attr:`tp_traverse` + a non-zero value, then return it. Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers look like:: static int @@ -151,7 +151,7 @@ must name its arguments exactly *visit* and *arg*: .. versionadded:: 2.4 -The :attr:`tp_clear` handler must be of the :c:type:`inquiry` type, or *NULL* +The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or *NULL* if the object is immutable. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index f5007ace020..e31687f554d 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -293,6 +293,6 @@ definition with the same method name. .. c:function:: PyObject* Py_FindMethod(PyMethodDef table[], PyObject *ob, char *name) Return a bound method object for an extension type implemented in C. This - can be useful in the implementation of a :attr:`tp_getattro` or - :attr:`tp_getattr` handler that does not use the + can be useful in the implementation of a :c:member:`~PyTypeObject.tp_getattro` or + :c:member:`~PyTypeObject.tp_getattr` handler that does not use the :c:func:`PyObject_GenericGetAttr` function. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 7c37786ddf9..c35f7277072 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -35,7 +35,7 @@ definition found there: The type object structure extends the :c:type:`PyVarObject` structure. The :attr:`ob_size` field is used for dynamic types (created by :func:`type_new`, usually called from a class statement). Note that :c:data:`PyType_Type` (the -metatype) initializes :attr:`tp_itemsize`, which means that its instances (i.e. +metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that its instances (i.e. type objects) *must* have the :attr:`ob_size` field. @@ -108,7 +108,7 @@ type objects) *must* have the :attr:`ob_size` field. should be just the type name. If the module is a submodule of a package, the full package name is part of the full module name. For example, a type named :class:`T` defined in module :mod:`M` in subpackage :mod:`Q` in package :mod:`P` - should have the :attr:`tp_name` initializer ``"P.Q.M.T"``. + should have the :c:member:`~PyTypeObject.tp_name` initializer ``"P.Q.M.T"``. For dynamically allocated type objects, this should just be the type name, and the module name explicitly stored in the type dict as the value for key @@ -119,7 +119,7 @@ type objects) *must* have the :attr:`ob_size` field. attribute, and everything after the last dot is made accessible as the :attr:`__name__` attribute. - If no dot is present, the entire :attr:`tp_name` field is made accessible as the + If no dot is present, the entire :c:member:`~PyTypeObject.tp_name` field is made accessible as the :attr:`__name__` attribute, and the :attr:`__module__` attribute is undefined (unless explicitly set in the dictionary, as explained above). This means your type will be impossible to pickle. @@ -133,13 +133,13 @@ type objects) *must* have the :attr:`ob_size` field. These fields allow calculating the size in bytes of instances of the type. There are two kinds of types: types with fixed-length instances have a zero - :attr:`tp_itemsize` field, types with variable-length instances have a non-zero - :attr:`tp_itemsize` field. For a type with fixed-length instances, all - instances have the same size, given in :attr:`tp_basicsize`. + :c:member:`~PyTypeObject.tp_itemsize` field, types with variable-length instances have a non-zero + :c:member:`~PyTypeObject.tp_itemsize` field. For a type with fixed-length instances, all + instances have the same size, given in :c:member:`~PyTypeObject.tp_basicsize`. For a type with variable-length instances, the instances must have an - :attr:`ob_size` field, and the instance size is :attr:`tp_basicsize` plus N - times :attr:`tp_itemsize`, where N is the "length" of the object. The value of + :attr:`ob_size` field, and the instance size is :c:member:`~PyTypeObject.tp_basicsize` plus N + times :c:member:`~PyTypeObject.tp_itemsize`, where N is the "length" of the object. The value of N is typically stored in the instance's :attr:`ob_size` field. There are exceptions: for example, long ints use a negative :attr:`ob_size` to indicate a negative number, and N is ``abs(ob_size)`` there. Also, the presence of an @@ -152,21 +152,21 @@ type objects) *must* have the :attr:`ob_size` field. :c:macro:`PyObject_HEAD` or :c:macro:`PyObject_VAR_HEAD` (whichever is used to declare the instance struct) and this in turn includes the :attr:`_ob_prev` and :attr:`_ob_next` fields if they are present. This means that the only correct - way to get an initializer for the :attr:`tp_basicsize` is to use the + way to get an initializer for the :c:member:`~PyTypeObject.tp_basicsize` is to use the ``sizeof`` operator on the struct used to declare the instance layout. The basic size does not include the GC header size (this is new in Python 2.2; - in 2.1 and 2.0, the GC header size was included in :attr:`tp_basicsize`). + in 2.1 and 2.0, the GC header size was included in :c:member:`~PyTypeObject.tp_basicsize`). These fields are inherited separately by subtypes. If the base type has a - non-zero :attr:`tp_itemsize`, it is generally not safe to set - :attr:`tp_itemsize` to a different non-zero value in a subtype (though this + non-zero :c:member:`~PyTypeObject.tp_itemsize`, it is generally not safe to set + :c:member:`~PyTypeObject.tp_itemsize` to a different non-zero value in a subtype (though this depends on the implementation of the base type). A note about alignment: if the variable items require a particular alignment, - this should be taken care of by the value of :attr:`tp_basicsize`. Example: - suppose a type implements an array of ``double``. :attr:`tp_itemsize` is + this should be taken care of by the value of :c:member:`~PyTypeObject.tp_basicsize`. Example: + suppose a type implements an array of ``double``. :c:member:`~PyTypeObject.tp_itemsize` is ``sizeof(double)``. It is the programmer's responsibility that - :attr:`tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the + :c:member:`~PyTypeObject.tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the alignment requirement for ``double``). @@ -182,10 +182,10 @@ type objects) *must* have the :attr:`ob_size` field. destructor function should free all references which the instance owns, free all memory buffers owned by the instance (using the freeing function corresponding to the allocation function used to allocate the buffer), and finally (as its - last action) call the type's :attr:`tp_free` function. If the type is not + last action) call the type's :c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is permissible to call the object deallocator directly instead of via - :attr:`tp_free`. The object deallocator should be the one used to allocate the + :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the instance; this is normally :c:func:`PyObject_Del` if the instance was allocated using :c:func:`PyObject_New` or :c:func:`PyObject_VarNew`, or :c:func:`PyObject_GC_Del` if the instance was allocated using @@ -200,25 +200,25 @@ type objects) *must* have the :attr:`ob_size` field. The print function is only called when the instance is printed to a *real* file; when it is printed to a pseudo-file (like a :class:`StringIO` instance), the - instance's :attr:`tp_repr` or :attr:`tp_str` function is called to convert it to - a string. These are also called when the type's :attr:`tp_print` field is - *NULL*. A type should never implement :attr:`tp_print` in a way that produces - different output than :attr:`tp_repr` or :attr:`tp_str` would. + instance's :c:member:`~PyTypeObject.tp_repr` or :c:member:`~PyTypeObject.tp_str` function is called to convert it to + a string. These are also called when the type's :c:member:`~PyTypeObject.tp_print` field is + *NULL*. A type should never implement :c:member:`~PyTypeObject.tp_print` in a way that produces + different output than :c:member:`~PyTypeObject.tp_repr` or :c:member:`~PyTypeObject.tp_str` would. The print function is called with the same signature as :c:func:`PyObject_Print`: ``int tp_print(PyObject *self, FILE *file, int flags)``. The *self* argument is the instance to be printed. The *file* argument is the stdio file to which it is to be printed. The *flags* argument is composed of flag bits. The only flag bit currently defined is :const:`Py_PRINT_RAW`. When the :const:`Py_PRINT_RAW` - flag bit is set, the instance should be printed the same way as :attr:`tp_str` + flag bit is set, the instance should be printed the same way as :c:member:`~PyTypeObject.tp_str` would format it; when the :const:`Py_PRINT_RAW` flag bit is clear, the instance - should be printed the same was as :attr:`tp_repr` would format it. It should + should be printed the same was as :c:member:`~PyTypeObject.tp_repr` would format it. It should return ``-1`` and set an exception condition when an error occurred during the comparison. - It is possible that the :attr:`tp_print` field will be deprecated. In any case, - it is recommended not to define :attr:`tp_print`, but instead to rely on - :attr:`tp_repr` and :attr:`tp_str` for printing. + It is possible that the :c:member:`~PyTypeObject.tp_print` field will be deprecated. In any case, + it is recommended not to define :c:member:`~PyTypeObject.tp_print`, but instead to rely on + :c:member:`~PyTypeObject.tp_repr` and :c:member:`~PyTypeObject.tp_str` for printing. This field is inherited by subtypes. @@ -228,13 +228,13 @@ type objects) *must* have the :attr:`ob_size` field. An optional pointer to the get-attribute-string function. This field is deprecated. When it is defined, it should point to a function - that acts the same as the :attr:`tp_getattro` function, but taking a C string + that acts the same as the :c:member:`~PyTypeObject.tp_getattro` function, but taking a C string instead of a Python string object to give the attribute name. The signature is the same as for :c:func:`PyObject_GetAttrString`. - This field is inherited by subtypes together with :attr:`tp_getattro`: a subtype - inherits both :attr:`tp_getattr` and :attr:`tp_getattro` from its base type when - the subtype's :attr:`tp_getattr` and :attr:`tp_getattro` are both *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattro`: a subtype + inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when + the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both *NULL*. .. c:member:: setattrfunc PyTypeObject.tp_setattr @@ -242,13 +242,13 @@ type objects) *must* have the :attr:`ob_size` field. An optional pointer to the set-attribute-string function. This field is deprecated. When it is defined, it should point to a function - that acts the same as the :attr:`tp_setattro` function, but taking a C string + that acts the same as the :c:member:`~PyTypeObject.tp_setattro` function, but taking a C string instead of a Python string object to give the attribute name. The signature is the same as for :c:func:`PyObject_SetAttrString`. - This field is inherited by subtypes together with :attr:`tp_setattro`: a subtype - inherits both :attr:`tp_setattr` and :attr:`tp_setattro` from its base type when - the subtype's :attr:`tp_setattr` and :attr:`tp_setattro` are both *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattro`: a subtype + inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when + the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*. .. c:member:: cmpfunc PyTypeObject.tp_compare @@ -260,10 +260,10 @@ type objects) *must* have the :attr:`ob_size` field. *other*, and ``-1`` if *self* less than *other*. It should return ``-1`` and set an exception condition when an error occurred during the comparison. - This field is inherited by subtypes together with :attr:`tp_richcompare` and - :attr:`tp_hash`: a subtypes inherits all three of :attr:`tp_compare`, - :attr:`tp_richcompare`, and :attr:`tp_hash` when the subtype's - :attr:`tp_compare`, :attr:`tp_richcompare`, and :attr:`tp_hash` are all *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_richcompare` and + :c:member:`~PyTypeObject.tp_hash`: a subtypes inherits all three of :c:member:`~PyTypeObject.tp_compare`, + :c:member:`~PyTypeObject.tp_richcompare`, and :c:member:`~PyTypeObject.tp_hash` when the subtype's + :c:member:`~PyTypeObject.tp_compare`, :c:member:`~PyTypeObject.tp_richcompare`, and :c:member:`~PyTypeObject.tp_hash` are all *NULL*. .. c:member:: reprfunc PyTypeObject.tp_repr @@ -292,7 +292,7 @@ type objects) *must* have the :attr:`ob_size` field. objects which implement the number protocol. These fields are documented in :ref:`number-structs`. - The :attr:`tp_as_number` field is not inherited, but the contained fields are + The :c:member:`~PyTypeObject.tp_as_number` field is not inherited, but the contained fields are inherited individually. @@ -302,7 +302,7 @@ type objects) *must* have the :attr:`ob_size` field. objects which implement the sequence protocol. These fields are documented in :ref:`sequence-structs`. - The :attr:`tp_as_sequence` field is not inherited, but the contained fields + The :c:member:`~PyTypeObject.tp_as_sequence` field is not inherited, but the contained fields are inherited individually. @@ -312,7 +312,7 @@ type objects) *must* have the :attr:`ob_size` field. objects which implement the mapping protocol. These fields are documented in :ref:`mapping-structs`. - The :attr:`tp_as_mapping` field is not inherited, but the contained fields + The :c:member:`~PyTypeObject.tp_as_mapping` field is not inherited, but the contained fields are inherited individually. @@ -336,14 +336,14 @@ type objects) *must* have the :attr:`ob_size` field. the Python level will result in the ``tp_hash`` slot being set to :c:func:`PyObject_HashNotImplemented`. - When this field is not set, two possibilities exist: if the :attr:`tp_compare` - and :attr:`tp_richcompare` fields are both *NULL*, a default hash value based on + When this field is not set, two possibilities exist: if the :c:member:`~PyTypeObject.tp_compare` + and :c:member:`~PyTypeObject.tp_richcompare` fields are both *NULL*, a default hash value based on the object's address is returned; otherwise, a :exc:`TypeError` is raised. - This field is inherited by subtypes together with :attr:`tp_richcompare` and - :attr:`tp_compare`: a subtypes inherits all three of :attr:`tp_compare`, - :attr:`tp_richcompare`, and :attr:`tp_hash`, when the subtype's - :attr:`tp_compare`, :attr:`tp_richcompare` and :attr:`tp_hash` are all *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_richcompare` and + :c:member:`~PyTypeObject.tp_compare`: a subtypes inherits all three of :c:member:`~PyTypeObject.tp_compare`, + :c:member:`~PyTypeObject.tp_richcompare`, and :c:member:`~PyTypeObject.tp_hash`, when the subtype's + :c:member:`~PyTypeObject.tp_compare`, :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are all *NULL*. .. c:member:: ternaryfunc PyTypeObject.tp_call @@ -381,9 +381,9 @@ type objects) *must* have the :attr:`ob_size` field. convenient to set this field to :c:func:`PyObject_GenericGetAttr`, which implements the normal way of looking for object attributes. - This field is inherited by subtypes together with :attr:`tp_getattr`: a subtype - inherits both :attr:`tp_getattr` and :attr:`tp_getattro` from its base type when - the subtype's :attr:`tp_getattr` and :attr:`tp_getattro` are both *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattr`: a subtype + inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when + the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both *NULL*. .. c:member:: setattrofunc PyTypeObject.tp_setattro @@ -394,9 +394,9 @@ type objects) *must* have the :attr:`ob_size` field. convenient to set this field to :c:func:`PyObject_GenericSetAttr`, which implements the normal way of setting object attributes. - This field is inherited by subtypes together with :attr:`tp_setattr`: a subtype - inherits both :attr:`tp_setattr` and :attr:`tp_setattro` from its base type when - the subtype's :attr:`tp_setattr` and :attr:`tp_setattro` are both *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattr`: a subtype + inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when + the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*. .. c:member:: PyBufferProcs* PyTypeObject.tp_as_buffer @@ -405,7 +405,7 @@ type objects) *must* have the :attr:`ob_size` field. which implement the buffer interface. These fields are documented in :ref:`buffer-structs`. - The :attr:`tp_as_buffer` field is not inherited, but the contained fields are + The :c:member:`~PyTypeObject.tp_as_buffer` field is not inherited, but the contained fields are inherited individually. @@ -414,8 +414,8 @@ type objects) *must* have the :attr:`ob_size` field. This field is a bit mask of various flags. Some flags indicate variant semantics for certain situations; others are used to indicate that certain fields in the type object (or in the extension structures referenced via - :attr:`tp_as_number`, :attr:`tp_as_sequence`, :attr:`tp_as_mapping`, and - :attr:`tp_as_buffer`) that were historically not always present are valid; if + :c:member:`~PyTypeObject.tp_as_number`, :c:member:`~PyTypeObject.tp_as_sequence`, :c:member:`~PyTypeObject.tp_as_mapping`, and + :c:member:`~PyTypeObject.tp_as_buffer`) that were historically not always present are valid; if such a flag bit is clear, the type fields it guards must not be accessed and must be considered to have a zero or *NULL* value instead. @@ -425,14 +425,14 @@ type objects) *must* have the :attr:`ob_size` field. inherited if the extension structure is inherited, i.e. the base type's value of the flag bit is copied into the subtype together with a pointer to the extension structure. The :const:`Py_TPFLAGS_HAVE_GC` flag bit is inherited together with - the :attr:`tp_traverse` and :attr:`tp_clear` fields, i.e. if the + the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields, i.e. if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the - :attr:`tp_traverse` and :attr:`tp_clear` fields in the subtype exist (as + :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist (as indicated by the :const:`Py_TPFLAGS_HAVE_RICHCOMPARE` flag bit) and have *NULL* values. The following bit masks are currently defined; these can be ORed together using - the ``|`` operator to form the value of the :attr:`tp_flags` field. The macro + the ``|`` operator to form the value of the :c:member:`~PyTypeObject.tp_flags` field. The macro :c:func:`PyType_HasFeature` takes a type and a flags value, *tp* and *f*, and checks whether ``tp->tp_flags & f`` is non-zero. @@ -440,13 +440,13 @@ type objects) *must* have the :attr:`ob_size` field. .. data:: Py_TPFLAGS_HAVE_GETCHARBUFFER If this bit is set, the :c:type:`PyBufferProcs` struct referenced by - :attr:`tp_as_buffer` has the :attr:`bf_getcharbuffer` field. + :c:member:`~PyTypeObject.tp_as_buffer` has the :attr:`bf_getcharbuffer` field. .. data:: Py_TPFLAGS_HAVE_SEQUENCE_IN If this bit is set, the :c:type:`PySequenceMethods` struct referenced by - :attr:`tp_as_sequence` has the :attr:`sq_contains` field. + :c:member:`~PyTypeObject.tp_as_sequence` has the :attr:`sq_contains` field. .. data:: Py_TPFLAGS_GC @@ -458,8 +458,8 @@ type objects) *must* have the :attr:`ob_size` field. .. data:: Py_TPFLAGS_HAVE_INPLACEOPS If this bit is set, the :c:type:`PySequenceMethods` struct referenced by - :attr:`tp_as_sequence` and the :c:type:`PyNumberMethods` structure referenced by - :attr:`tp_as_number` contain the fields for in-place operators. In particular, + :c:member:`~PyTypeObject.tp_as_sequence` and the :c:type:`PyNumberMethods` structure referenced by + :c:member:`~PyTypeObject.tp_as_number` contain the fields for in-place operators. In particular, this means that the :c:type:`PyNumberMethods` structure has the fields :attr:`nb_inplace_add`, :attr:`nb_inplace_subtract`, :attr:`nb_inplace_multiply`, :attr:`nb_inplace_divide`, @@ -473,7 +473,7 @@ type objects) *must* have the :attr:`ob_size` field. .. data:: Py_TPFLAGS_CHECKTYPES If this bit is set, the binary and ternary operations in the - :c:type:`PyNumberMethods` structure referenced by :attr:`tp_as_number` accept + :c:type:`PyNumberMethods` structure referenced by :c:member:`~PyTypeObject.tp_as_number` accept arguments of arbitrary object types, and do their own type conversions if needed. If this bit is clear, those operations require that all arguments have the current type as their type, and the caller is supposed to perform a coercion @@ -485,31 +485,31 @@ type objects) *must* have the :attr:`ob_size` field. .. data:: Py_TPFLAGS_HAVE_RICHCOMPARE - If this bit is set, the type object has the :attr:`tp_richcompare` field, as - well as the :attr:`tp_traverse` and the :attr:`tp_clear` fields. + If this bit is set, the type object has the :c:member:`~PyTypeObject.tp_richcompare` field, as + well as the :c:member:`~PyTypeObject.tp_traverse` and the :c:member:`~PyTypeObject.tp_clear` fields. .. data:: Py_TPFLAGS_HAVE_WEAKREFS - If this bit is set, the :attr:`tp_weaklistoffset` field is defined. Instances - of a type are weakly referenceable if the type's :attr:`tp_weaklistoffset` field + If this bit is set, the :c:member:`~PyTypeObject.tp_weaklistoffset` field is defined. Instances + of a type are weakly referenceable if the type's :c:member:`~PyTypeObject.tp_weaklistoffset` field has a value greater than zero. .. data:: Py_TPFLAGS_HAVE_ITER - If this bit is set, the type object has the :attr:`tp_iter` and - :attr:`tp_iternext` fields. + If this bit is set, the type object has the :c:member:`~PyTypeObject.tp_iter` and + :c:member:`~PyTypeObject.tp_iternext` fields. .. data:: Py_TPFLAGS_HAVE_CLASS If this bit is set, the type object has several new fields defined starting in - Python 2.2: :attr:`tp_methods`, :attr:`tp_members`, :attr:`tp_getset`, - :attr:`tp_base`, :attr:`tp_dict`, :attr:`tp_descr_get`, :attr:`tp_descr_set`, - :attr:`tp_dictoffset`, :attr:`tp_init`, :attr:`tp_alloc`, :attr:`tp_new`, - :attr:`tp_free`, :attr:`tp_is_gc`, :attr:`tp_bases`, :attr:`tp_mro`, - :attr:`tp_cache`, :attr:`tp_subclasses`, and :attr:`tp_weaklist`. + Python 2.2: :c:member:`~PyTypeObject.tp_methods`, :c:member:`~PyTypeObject.tp_members`, :c:member:`~PyTypeObject.tp_getset`, + :c:member:`~PyTypeObject.tp_base`, :c:member:`~PyTypeObject.tp_dict`, :c:member:`~PyTypeObject.tp_descr_get`, :c:member:`~PyTypeObject.tp_descr_set`, + :c:member:`~PyTypeObject.tp_dictoffset`, :c:member:`~PyTypeObject.tp_init`, :c:member:`~PyTypeObject.tp_alloc`, :c:member:`~PyTypeObject.tp_new`, + :c:member:`~PyTypeObject.tp_free`, :c:member:`~PyTypeObject.tp_is_gc`, :c:member:`~PyTypeObject.tp_bases`, :c:member:`~PyTypeObject.tp_mro`, + :c:member:`~PyTypeObject.tp_cache`, :c:member:`~PyTypeObject.tp_subclasses`, and :c:member:`~PyTypeObject.tp_weaklist`. .. data:: Py_TPFLAGS_HEAPTYPE @@ -547,7 +547,7 @@ type objects) *must* have the :attr:`ob_size` field. is set, instances must be created using :c:func:`PyObject_GC_New` and destroyed using :c:func:`PyObject_GC_Del`. More information in section :ref:`supporting-cycle-detection`. This bit also implies that the - GC-related fields :attr:`tp_traverse` and :attr:`tp_clear` are present in + GC-related fields :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` are present in the type object; but those fields also exist when :const:`Py_TPFLAGS_HAVE_GC` is clear but :const:`Py_TPFLAGS_HAVE_RICHCOMPARE` is set. @@ -582,8 +582,8 @@ The following three fields only exist if the about Python's garbage collection scheme can be found in section :ref:`supporting-cycle-detection`. - The :attr:`tp_traverse` pointer is used by the garbage collector to detect - reference cycles. A typical implementation of a :attr:`tp_traverse` function + The :c:member:`~PyTypeObject.tp_traverse` pointer is used by the garbage collector to detect + reference cycles. A typical implementation of a :c:member:`~PyTypeObject.tp_traverse` function simply calls :c:func:`Py_VISIT` on each of the instance's members that are Python objects. For example, this is function :c:func:`local_traverse` from the :mod:`thread` extension module:: @@ -609,9 +609,9 @@ The following three fields only exist if the :c:func:`local_traverse` to have these specific names; don't name them just anything. - This field is inherited by subtypes together with :attr:`tp_clear` and the - :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :attr:`tp_traverse`, and - :attr:`tp_clear` are all inherited from the base type if they are all zero in + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_clear` and the + :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and + :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in the subtype *and* the subtype has the :const:`Py_TPFLAGS_HAVE_RICHCOMPARE` flag bit set. @@ -621,17 +621,17 @@ The following three fields only exist if the An optional pointer to a clear function for the garbage collector. This is only used if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is set. - The :attr:`tp_clear` member function is used to break reference cycles in cyclic - garbage detected by the garbage collector. Taken together, all :attr:`tp_clear` + The :c:member:`~PyTypeObject.tp_clear` member function is used to break reference cycles in cyclic + garbage detected by the garbage collector. Taken together, all :c:member:`~PyTypeObject.tp_clear` functions in the system must combine to break all reference cycles. This is - subtle, and if in any doubt supply a :attr:`tp_clear` function. For example, - the tuple type does not implement a :attr:`tp_clear` function, because it's + subtle, and if in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For example, + the tuple type does not implement a :c:member:`~PyTypeObject.tp_clear` function, because it's possible to prove that no reference cycle can be composed entirely of tuples. - Therefore the :attr:`tp_clear` functions of other types must be sufficient to + Therefore the :c:member:`~PyTypeObject.tp_clear` functions of other types must be sufficient to break any cycle containing a tuple. This isn't immediately obvious, and there's - rarely a good reason to avoid implementing :attr:`tp_clear`. + rarely a good reason to avoid implementing :c:member:`~PyTypeObject.tp_clear`. - Implementations of :attr:`tp_clear` should drop the instance's references to + Implementations of :c:member:`~PyTypeObject.tp_clear` should drop the instance's references to those of its members that may be Python objects, and set its pointers to those members to *NULL*, as in the following example:: @@ -656,18 +656,18 @@ The following three fields only exist if the so that *self* knows the contained object can no longer be used. The :c:func:`Py_CLEAR` macro performs the operations in a safe order. - Because the goal of :attr:`tp_clear` functions is to break reference cycles, + Because the goal of :c:member:`~PyTypeObject.tp_clear` functions is to break reference cycles, it's not necessary to clear contained objects like Python strings or Python integers, which can't participate in reference cycles. On the other hand, it may be convenient to clear all contained Python objects, and write the type's - :attr:`tp_dealloc` function to invoke :attr:`tp_clear`. + :c:member:`~PyTypeObject.tp_dealloc` function to invoke :c:member:`~PyTypeObject.tp_clear`. More information about Python's garbage collection scheme can be found in section :ref:`supporting-cycle-detection`. - This field is inherited by subtypes together with :attr:`tp_traverse` and the - :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :attr:`tp_traverse`, and - :attr:`tp_clear` are all inherited from the base type if they are all zero in + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_traverse` and the + :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and + :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in the subtype *and* the subtype has the :const:`Py_TPFLAGS_HAVE_RICHCOMPARE` flag bit set. @@ -688,13 +688,13 @@ The following three fields only exist if the comparisons makes sense (e.g. ``==`` and ``!=``, but not ``<`` and friends), directly raise :exc:`TypeError` in the rich comparison function. - This field is inherited by subtypes together with :attr:`tp_compare` and - :attr:`tp_hash`: a subtype inherits all three of :attr:`tp_compare`, - :attr:`tp_richcompare`, and :attr:`tp_hash`, when the subtype's - :attr:`tp_compare`, :attr:`tp_richcompare`, and :attr:`tp_hash` are all *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_compare` and + :c:member:`~PyTypeObject.tp_hash`: a subtype inherits all three of :c:member:`~PyTypeObject.tp_compare`, + :c:member:`~PyTypeObject.tp_richcompare`, and :c:member:`~PyTypeObject.tp_hash`, when the subtype's + :c:member:`~PyTypeObject.tp_compare`, :c:member:`~PyTypeObject.tp_richcompare`, and :c:member:`~PyTypeObject.tp_hash` are all *NULL*. The following constants are defined to be used as the third argument for - :attr:`tp_richcompare` and for :c:func:`PyObject_RichCompare`: + :c:member:`~PyTypeObject.tp_richcompare` and for :c:func:`PyObject_RichCompare`: +----------------+------------+ | Constant | Comparison | @@ -725,26 +725,26 @@ set. instance structure needs to include a field of type :c:type:`PyObject\*` which is initialized to *NULL*. - Do not confuse this field with :attr:`tp_weaklist`; that is the list head for + Do not confuse this field with :c:member:`~PyTypeObject.tp_weaklist`; that is the list head for weak references to the type object itself. This field is inherited by subtypes, but see the rules listed below. A subtype may override this offset; this means that the subtype uses a different weak reference list head than the base type. Since the list head is always found via - :attr:`tp_weaklistoffset`, this should not be a problem. + :c:member:`~PyTypeObject.tp_weaklistoffset`, this should not be a problem. When a type defined by a class statement has no :attr:`__slots__` declaration, and none of its base types are weakly referenceable, the type is made weakly referenceable by adding a weak reference list head slot to the instance layout - and setting the :attr:`tp_weaklistoffset` of that slot's offset. + and setting the :c:member:`~PyTypeObject.tp_weaklistoffset` of that slot's offset. When a type's :attr:`__slots__` declaration contains a slot named :attr:`__weakref__`, that slot becomes the weak reference list head for instances of the type, and the slot's offset is stored in the type's - :attr:`tp_weaklistoffset`. + :c:member:`~PyTypeObject.tp_weaklistoffset`. When a type's :attr:`__slots__` declaration does not contain a slot named - :attr:`__weakref__`, the type inherits its :attr:`tp_weaklistoffset` from its + :attr:`__weakref__`, the type inherits its :c:member:`~PyTypeObject.tp_weaklistoffset` from its base type. The next two fields only exist if the :const:`Py_TPFLAGS_HAVE_ITER` flag bit is @@ -772,7 +772,7 @@ set. are iterators (although classic instances always have this function, even if they don't define a :meth:`next` method). - Iterator types should also define the :attr:`tp_iter` function, and that + Iterator types should also define the :c:member:`~PyTypeObject.tp_iter` function, and that function should return the iterator instance itself (not a new iterator instance). @@ -780,7 +780,7 @@ set. This field is inherited by subtypes. -The next fields, up to and including :attr:`tp_weaklist`, only exist if the +The next fields, up to and including :c:member:`~PyTypeObject.tp_weaklist`, only exist if the :const:`Py_TPFLAGS_HAVE_CLASS` flag bit is set. @@ -790,7 +790,7 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the structures, declaring regular methods of this type. For each entry in the array, an entry is added to the type's dictionary (see - :attr:`tp_dict` below) containing a method descriptor. + :c:member:`~PyTypeObject.tp_dict` below) containing a method descriptor. This field is not inherited by subtypes (methods are inherited through a different mechanism). @@ -803,7 +803,7 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the this type. For each entry in the array, an entry is added to the type's dictionary (see - :attr:`tp_dict` below) containing a member descriptor. + :c:member:`~PyTypeObject.tp_dict` below) containing a member descriptor. This field is not inherited by subtypes (members are inherited through a different mechanism). @@ -815,7 +815,7 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the structures, declaring computed attributes of instances of this type. For each entry in the array, an entry is added to the type's dictionary (see - :attr:`tp_dict` below) containing a getset descriptor. + :c:member:`~PyTypeObject.tp_dict` below) containing a getset descriptor. This field is not inherited by subtypes (computed attributes are inherited through a different mechanism). @@ -894,7 +894,7 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the the instance variable dictionary; this offset is used by :c:func:`PyObject_GenericGetAttr`. - Do not confuse this field with :attr:`tp_dict`; that is the dictionary for + Do not confuse this field with :c:member:`~PyTypeObject.tp_dict`; that is the dictionary for attributes of the type object itself. If the value of this field is greater than zero, it specifies the offset from @@ -903,20 +903,20 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the offset is more expensive to use, and should only be used when the instance structure contains a variable-length part. This is used for example to add an instance variable dictionary to subtypes of :class:`str` or :class:`tuple`. Note - that the :attr:`tp_basicsize` field should account for the dictionary added to + that the :c:member:`~PyTypeObject.tp_basicsize` field should account for the dictionary added to the end in that case, even though the dictionary is not included in the basic object layout. On a system with a pointer size of 4 bytes, - :attr:`tp_dictoffset` should be set to ``-4`` to indicate that the dictionary is + :c:member:`~PyTypeObject.tp_dictoffset` should be set to ``-4`` to indicate that the dictionary is at the very end of the structure. The real dictionary offset in an instance can be computed from a negative - :attr:`tp_dictoffset` as follows:: + :c:member:`~PyTypeObject.tp_dictoffset` as follows:: dictoffset = tp_basicsize + abs(ob_size)*tp_itemsize + tp_dictoffset if dictoffset is not aligned on sizeof(void*): round up to sizeof(void*) - where :attr:`tp_basicsize`, :attr:`tp_itemsize` and :attr:`tp_dictoffset` are + where :c:member:`~PyTypeObject.tp_basicsize`, :c:member:`~PyTypeObject.tp_itemsize` and :c:member:`~PyTypeObject.tp_dictoffset` are taken from the type object, and :attr:`ob_size` is taken from the instance. The absolute value is taken because long ints use the sign of :attr:`ob_size` to store the sign of the number. (There's never a need to do this calculation @@ -925,15 +925,15 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the This field is inherited by subtypes, but see the rules listed below. A subtype may override this offset; this means that the subtype instances store the dictionary at a difference offset than the base type. Since the dictionary is - always found via :attr:`tp_dictoffset`, this should not be a problem. + always found via :c:member:`~PyTypeObject.tp_dictoffset`, this should not be a problem. When a type defined by a class statement has no :attr:`__slots__` declaration, and none of its base types has an instance variable dictionary, a dictionary - slot is added to the instance layout and the :attr:`tp_dictoffset` is set to + slot is added to the instance layout and the :c:member:`~PyTypeObject.tp_dictoffset` is set to that slot's offset. When a type defined by a class statement has a :attr:`__slots__` declaration, - the type inherits its :attr:`tp_dictoffset` from its base type. + the type inherits its :c:member:`~PyTypeObject.tp_dictoffset` from its base type. (Adding a slot named :attr:`__dict__` to the :attr:`__slots__` declaration does not have the expected effect, it just causes confusion. Maybe this should be @@ -957,15 +957,15 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the arguments represent positional and keyword arguments of the call to :meth:`__init__`. - The :attr:`tp_init` function, if not *NULL*, is called when an instance is - created normally by calling its type, after the type's :attr:`tp_new` function - has returned an instance of the type. If the :attr:`tp_new` function returns an + The :c:member:`~PyTypeObject.tp_init` function, if not *NULL*, is called when an instance is + created normally by calling its type, after the type's :c:member:`~PyTypeObject.tp_new` function + has returned an instance of the type. If the :c:member:`~PyTypeObject.tp_new` function returns an instance of some other type that is not a subtype of the original type, no - :attr:`tp_init` function is called; if :attr:`tp_new` returns an instance of a - subtype of the original type, the subtype's :attr:`tp_init` is called. (VERSION + :c:member:`~PyTypeObject.tp_init` function is called; if :c:member:`~PyTypeObject.tp_new` returns an instance of a + subtype of the original type, the subtype's :c:member:`~PyTypeObject.tp_init` is called. (VERSION NOTE: described here is what is implemented in Python 2.2.1 and later. In - Python 2.2, the :attr:`tp_init` of the type of the object returned by - :attr:`tp_new` was always called, if not *NULL*.) + Python 2.2, the :c:member:`~PyTypeObject.tp_init` of the type of the object returned by + :c:member:`~PyTypeObject.tp_new` was always called, if not *NULL*.) This field is inherited by subtypes. @@ -982,14 +982,14 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the initialization. It should return a pointer to a block of memory of adequate length for the instance, suitably aligned, and initialized to zeros, but with :attr:`ob_refcnt` set to ``1`` and :attr:`ob_type` set to the type argument. If - the type's :attr:`tp_itemsize` is non-zero, the object's :attr:`ob_size` field + the type's :c:member:`~PyTypeObject.tp_itemsize` is non-zero, the object's :attr:`ob_size` field should be initialized to *nitems* and the length of the allocated memory block should be ``tp_basicsize + nitems*tp_itemsize``, rounded up to a multiple of ``sizeof(void*)``; otherwise, *nitems* is not used and the length of the block - should be :attr:`tp_basicsize`. + should be :c:member:`~PyTypeObject.tp_basicsize`. Do not use this function to do any other instance initialization, not even to - allocate additional memory; that should be done by :attr:`tp_new`. + allocate additional memory; that should be done by :c:member:`~PyTypeObject.tp_new`. This field is inherited by static subtypes, but not by dynamic subtypes (subtypes created by a class statement); in the latter, this field is always set @@ -1011,20 +1011,20 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the The subtype argument is the type of the object being created; the *args* and *kwds* arguments represent positional and keyword arguments of the call to the - type. Note that subtype doesn't have to equal the type whose :attr:`tp_new` + type. Note that subtype doesn't have to equal the type whose :c:member:`~PyTypeObject.tp_new` function is called; it may be a subtype of that type (but not an unrelated type). - The :attr:`tp_new` function should call ``subtype->tp_alloc(subtype, nitems)`` + The :c:member:`~PyTypeObject.tp_new` function should call ``subtype->tp_alloc(subtype, nitems)`` to allocate space for the object, and then do only as much further initialization as is absolutely necessary. Initialization that can safely be - ignored or repeated should be placed in the :attr:`tp_init` handler. A good + ignored or repeated should be placed in the :c:member:`~PyTypeObject.tp_init` handler. A good rule of thumb is that for immutable types, all initialization should take place - in :attr:`tp_new`, while for mutable types, most initialization should be - deferred to :attr:`tp_init`. + in :c:member:`~PyTypeObject.tp_new`, while for mutable types, most initialization should be + deferred to :c:member:`~PyTypeObject.tp_init`. This field is inherited by subtypes, except it is not inherited by static types - whose :attr:`tp_base` is *NULL* or ``&PyBaseObject_Type``. The latter exception + whose :c:member:`~PyTypeObject.tp_base` is *NULL* or ``&PyBaseObject_Type``. The latter exception is a precaution so that old extension types don't become callable simply by being linked with Python 2.2. @@ -1057,7 +1057,7 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the The garbage collector needs to know whether a particular object is collectible or not. Normally, it is sufficient to look at the object's type's - :attr:`tp_flags` field, and check the :const:`Py_TPFLAGS_HAVE_GC` flag bit. But + :c:member:`~PyTypeObject.tp_flags` field, and check the :const:`Py_TPFLAGS_HAVE_GC` flag bit. But some types have a mixture of statically and dynamically allocated instances, and the statically allocated instances are not collectible. Such types should define this function; it should return ``1`` for a collectible instance, and @@ -1129,7 +1129,7 @@ subtypes. .. c:member:: PyTypeObject* PyTypeObject.tp_next - Pointer to the next type object with a non-zero :attr:`tp_allocs` field. + Pointer to the next type object with a non-zero :c:member:`~PyTypeObject.tp_allocs` field. Also, note that, in a garbage collected Python, tp_dealloc may be called from any Python thread, not just the thread which created the object (if the object @@ -1289,13 +1289,13 @@ Sequence Object Structures This function is used by :c:func:`PySequence_Concat` and has the same signature. It is also used by the ``+`` operator, after trying the numeric - addition via the :attr:`tp_as_number.nb_add` slot. + addition via the :c:member:`~PyTypeObject.tp_as_number.nb_add` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_repeat This function is used by :c:func:`PySequence_Repeat` and has the same signature. It is also used by the ``*`` operator, after trying numeric - multiplication via the :attr:`tp_as_number.nb_mul` slot. + multiplication via the :c:member:`~PyTypeObject.tp_as_number.nb_mul` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_item @@ -1348,14 +1348,14 @@ data as a set of chunks of data, where each chunk is specified as a pointer/length pair. These chunks are called :dfn:`segments` and are presumed to be non-contiguous in memory. -If an object does not export the buffer interface, then its :attr:`tp_as_buffer` +If an object does not export the buffer interface, then its :c:member:`~PyTypeObject.tp_as_buffer` member in the :c:type:`PyTypeObject` structure should be *NULL*. Otherwise, the -:attr:`tp_as_buffer` will point to a :c:type:`PyBufferProcs` structure. +:c:member:`~PyTypeObject.tp_as_buffer` will point to a :c:type:`PyBufferProcs` structure. .. note:: It is very important that your :c:type:`PyTypeObject` structure uses - :const:`Py_TPFLAGS_DEFAULT` for the value of the :attr:`tp_flags` member rather + :const:`Py_TPFLAGS_DEFAULT` for the value of the :c:member:`~PyTypeObject.tp_flags` member rather than ``0``. This tells the Python runtime that your :c:type:`PyBufferProcs` structure contains the :attr:`bf_getcharbuffer` slot. Older versions of Python did not have this member, so a new Python interpreter using an old extension @@ -1385,7 +1385,7 @@ member in the :c:type:`PyTypeObject` structure should be *NULL*. Otherwise, the The last slot is :attr:`bf_getcharbuffer`, of type :c:type:`getcharbufferproc`. This slot will only be present if the :const:`Py_TPFLAGS_HAVE_GETCHARBUFFER` - flag is present in the :attr:`tp_flags` field of the object's + flag is present in the :c:member:`~PyTypeObject.tp_flags` field of the object's :c:type:`PyTypeObject`. Before using this slot, the caller should test whether it is present by using the :c:func:`PyType_HasFeature` function. If the flag is present, :attr:`bf_getcharbuffer` may be *NULL*, indicating that the object's diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 269c8fdd770..427821335c4 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -150,11 +150,11 @@ This is so that Python knows how much memory to allocate when you call .. note:: If you want your type to be subclassable from Python, and your type has the same - :attr:`tp_basicsize` as its base type, you may have problems with multiple + :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple inheritance. A Python subclass of your type will have to list your type first in its :attr:`__bases__`, or else it will not be able to call your type's :meth:`__new__` method without getting an error. You can avoid this problem by - ensuring that your type has a larger value for :attr:`tp_basicsize` than its + ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its base type does. Most of the time, this will be true anyway, because either your base type will be :class:`object`, or else you will be adding data members to your base type, and therefore increasing its size. @@ -174,7 +174,7 @@ to :const:`Py_TPFLAGS_DEFAULT`. :: All types should include this constant in their flags. It enables all of the members defined by the current version of Python. -We provide a doc string for the type in :attr:`tp_doc`. :: +We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. :: "Noddy objects", /* tp_doc */ @@ -183,12 +183,12 @@ from the others. We aren't going to implement any of these in this version of the module. We'll expand this example later to have more interesting behavior. For now, all we want to be able to do is to create new :class:`Noddy` objects. -To enable object creation, we have to provide a :attr:`tp_new` implementation. +To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new` implementation. In this case, we can just use the default implementation provided by the API function :c:func:`PyType_GenericNew`. We'd like to just assign this to the -:attr:`tp_new` slot, but we can't, for portability sake, On some platforms or +:c:member:`~PyTypeObject.tp_new` slot, but we can't, for portability sake, On some platforms or compilers, we can't statically initialize a structure member with a function -defined in another C module, so, instead, we'll assign the :attr:`tp_new` slot +defined in another C module, so, instead, we'll assign the :c:member:`~PyTypeObject.tp_new` slot in the module initialization function just before calling :c:func:`PyType_Ready`:: @@ -283,13 +283,13 @@ allocation and deallocation. At a minimum, we need a deallocation method:: self->ob_type->tp_free((PyObject*)self); } -which is assigned to the :attr:`tp_dealloc` member:: +which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member:: (destructor)Noddy_dealloc, /*tp_dealloc*/ This method decrements the reference counts of the two Python attributes. We use :c:func:`Py_XDECREF` here because the :attr:`first` and :attr:`last` members -could be *NULL*. It then calls the :attr:`tp_free` member of the object's type +could be *NULL*. It then calls the :c:member:`~PyTypeObject.tp_free` member of the object's type to free the object's memory. Note that the object's type might not be :class:`NoddyType`, because the object may be an instance of a subclass. @@ -323,7 +323,7 @@ strings, so we provide a new method:: return (PyObject *)self; } -and install it in the :attr:`tp_new` member:: +and install it in the :c:member:`~PyTypeObject.tp_new` member:: Noddy_new, /* tp_new */ @@ -344,16 +344,16 @@ created. New methods always accept positional and keyword arguments, but they often ignore the arguments, leaving the argument handling to initializer methods. Note that if the type supports subclassing, the type passed may not be the type being defined. The new method calls the tp_alloc slot to allocate -memory. We don't fill the :attr:`tp_alloc` slot ourselves. Rather +memory. We don't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather :c:func:`PyType_Ready` fills it for us by inheriting it from our base class, which is :class:`object` by default. Most types use the default allocation. .. note:: - If you are creating a co-operative :attr:`tp_new` (one that calls a base type's - :attr:`tp_new` or :meth:`__new__`), you must *not* try to determine what method + If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one that calls a base type's + :c:member:`~PyTypeObject.tp_new` or :meth:`__new__`), you must *not* try to determine what method to call using method resolution order at runtime. Always statically determine - what type you are going to call, and call its :attr:`tp_new` directly, or via + what type you are going to call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via ``type->tp_base->tp_new``. If you do not do this, Python subclasses of your type that also inherit from other Python-defined classes may not work correctly. (Specifically, you may not be able to create instances of such subclasses @@ -390,11 +390,11 @@ We provide an initialization function:: return 0; } -by filling the :attr:`tp_init` slot. :: +by filling the :c:member:`~PyTypeObject.tp_init` slot. :: (initproc)Noddy_init, /* tp_init */ -The :attr:`tp_init` slot is exposed in Python as the :meth:`__init__` method. It +The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the :meth:`__init__` method. It is used to initialize an object after it's created. Unlike the new method, we can't guarantee that the initializer is called. The initializer isn't called when unpickling objects and it can be overridden. Our initializer accepts @@ -424,7 +424,7 @@ reference counts. When don't we have to do this? * when we know that deallocation of the object [#]_ will not cause any calls back into our type's code -* when decrementing a reference count in a :attr:`tp_dealloc` handler when +* when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc` handler when garbage-collections is not supported [#]_ We want to expose our instance variables as attributes. There are a @@ -440,7 +440,7 @@ number of ways to do that. The simplest way is to define member definitions:: {NULL} /* Sentinel */ }; -and put the definitions in the :attr:`tp_members` slot:: +and put the definitions in the :c:member:`~PyTypeObject.tp_members` slot:: Noddy_members, /* tp_members */ @@ -516,7 +516,7 @@ definitions:: {NULL} /* Sentinel */ }; -and assign them to the :attr:`tp_methods` slot:: +and assign them to the :c:member:`~PyTypeObject.tp_methods` slot:: Noddy_methods, /* tp_methods */ @@ -611,7 +611,7 @@ We create an array of :c:type:`PyGetSetDef` structures:: {NULL} /* Sentinel */ }; -and register it in the :attr:`tp_getset` slot:: +and register it in the :c:member:`~PyTypeObject.tp_getset` slot:: Noddy_getseters, /* tp_getset */ @@ -628,7 +628,7 @@ We also remove the member definitions for these attributes:: {NULL} /* Sentinel */ }; -We also need to update the :attr:`tp_init` handler to only allow strings [#]_ to +We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only allow strings [#]_ to be passed:: static int @@ -747,7 +747,7 @@ simplified:: .. note:: - Note that the :attr:`tp_traverse` implementation must name its arguments exactly + Note that the :c:member:`~PyTypeObject.tp_traverse` implementation must name its arguments exactly *visit* and *arg* in order to use :c:func:`Py_VISIT`. This is to encourage uniformity across these boring implementations. @@ -784,7 +784,7 @@ its reference count. We do this because, as was discussed earlier, if the reference count drops to zero, we might cause code to run that calls back into the object. In addition, because we now support garbage collection, we also have to worry about code being run that triggers garbage collection. If garbage -collection is run, our :attr:`tp_traverse` handler could get called. We can't +collection is run, our :c:member:`~PyTypeObject.tp_traverse` handler could get called. We can't take a chance of having :c:func:`Noddy_traverse` called when a member's reference count has dropped to zero and its value hasn't been set to *NULL*. @@ -804,8 +804,8 @@ Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags:: Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ -That's pretty much it. If we had written custom :attr:`tp_alloc` or -:attr:`tp_free` slots, we'd need to modify them for cyclic-garbage collection. +That's pretty much it. If we had written custom :c:member:`~PyTypeObject.tp_alloc` or +:c:member:`~PyTypeObject.tp_free` slots, we'd need to modify them for cyclic-garbage collection. Most extensions will use the versions automatically provided. @@ -864,8 +864,8 @@ the :attr:`__init__` method of the base type. This pattern is important when writing a type with custom :attr:`new` and :attr:`dealloc` methods. The :attr:`new` method should not actually create the -memory for the object with :attr:`tp_alloc`, that will be handled by the base -class when calling its :attr:`tp_new`. +memory for the object with :c:member:`~PyTypeObject.tp_alloc`, that will be handled by the base +class when calling its :c:member:`~PyTypeObject.tp_new`. When filling out the :c:func:`PyTypeObject` for the :class:`Shoddy` type, you see a slot for :c:func:`tp_base`. Due to cross platform compiler issues, you can't @@ -890,8 +890,8 @@ the module's :c:func:`init` function. :: } Before calling :c:func:`PyType_Ready`, the type structure must have the -:attr:`tp_base` slot filled in. When we are deriving a new type, it is not -necessary to fill out the :attr:`tp_alloc` slot with :c:func:`PyType_GenericNew` +:c:member:`~PyTypeObject.tp_base` slot filled in. When we are deriving a new type, it is not +necessary to fill out the :c:member:`~PyTypeObject.tp_alloc` slot with :c:func:`PyType_GenericNew` -- the allocate function from the base type will be inherited. After that, calling :c:func:`PyType_Ready` and adding the type object to the @@ -934,7 +934,7 @@ that will be helpful in such a situation! :: These fields tell the runtime how much memory to allocate when new objects of this type are created. Python has some built-in support for variable length -structures (think: strings, lists) which is where the :attr:`tp_itemsize` field +structures (think: strings, lists) which is where the :c:member:`~PyTypeObject.tp_itemsize` field comes in. This will be dealt with later. :: char *tp_doc; @@ -1032,13 +1032,13 @@ that creating a temporary string object to be written to a file is too expensive. These handlers are all optional, and most types at most need to implement the -:attr:`tp_str` and :attr:`tp_repr` handlers. :: +:c:member:`~PyTypeObject.tp_str` and :c:member:`~PyTypeObject.tp_repr` handlers. :: reprfunc tp_repr; reprfunc tp_str; printfunc tp_print; -The :attr:`tp_repr` handler should return a string object containing a +The :c:member:`~PyTypeObject.tp_repr` handler should return a string object containing a representation of the instance for which it is called. Here is a simple example:: @@ -1049,15 +1049,15 @@ example:: obj->obj_UnderlyingDatatypePtr->size); } -If no :attr:`tp_repr` handler is specified, the interpreter will supply a -representation that uses the type's :attr:`tp_name` and a uniquely-identifying +If no :c:member:`~PyTypeObject.tp_repr` handler is specified, the interpreter will supply a +representation that uses the type's :c:member:`~PyTypeObject.tp_name` and a uniquely-identifying value for the object. -The :attr:`tp_str` handler is to :func:`str` what the :attr:`tp_repr` handler +The :c:member:`~PyTypeObject.tp_str` handler is to :func:`str` what the :c:member:`~PyTypeObject.tp_repr` handler described above is to :func:`repr`; that is, it is called when Python code calls :func:`str` on an instance of your object. Its implementation is very similar -to the :attr:`tp_repr` function, but the resulting string is intended for human -consumption. If :attr:`tp_str` is not specified, the :attr:`tp_repr` handler is +to the :c:member:`~PyTypeObject.tp_repr` function, but the resulting string is intended for human +consumption. If :c:member:`~PyTypeObject.tp_str` is not specified, the :c:member:`~PyTypeObject.tp_repr` handler is used instead. Here is a simple example:: @@ -1152,7 +1152,7 @@ type object to create :term:`descriptor`\s which are placed in the dictionary of type object. Each descriptor controls access to one attribute of the instance object. Each of the tables is optional; if all three are *NULL*, instances of the type will only have attributes that are inherited from their base type, and -should leave the :attr:`tp_getattro` and :attr:`tp_setattro` fields *NULL* as +should leave the :c:member:`~PyTypeObject.tp_getattro` and :c:member:`~PyTypeObject.tp_setattro` fields *NULL* as well, allowing the base type to handle attributes. The tables are declared as three fields of the type object:: @@ -1161,7 +1161,7 @@ The tables are declared as three fields of the type object:: struct PyMemberDef *tp_members; struct PyGetSetDef *tp_getset; -If :attr:`tp_methods` is not *NULL*, it must refer to an array of +If :c:member:`~PyTypeObject.tp_methods` is not *NULL*, it must refer to an array of :c:type:`PyMethodDef` structures. Each entry in the table is an instance of this structure:: @@ -1225,13 +1225,13 @@ combined using bitwise-OR. single: WRITE_RESTRICTED single: RESTRICTED -An interesting advantage of using the :attr:`tp_members` table to build +An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table to build descriptors that are used at runtime is that any attribute defined this way can have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the class object, and get the doc string using its :attr:`__doc__` attribute. -As with the :attr:`tp_methods` table, a sentinel entry with a :attr:`name` value +As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :attr:`name` value of *NULL* is required. .. XXX Descriptors need to be explained in more detail somewhere, but not here. @@ -1257,7 +1257,7 @@ portable to older versions of Python, and explains how the handler functions are called, so that if you do need to extend their functionality, you'll understand what needs to be done. -The :attr:`tp_getattr` handler is called when the object requires an attribute +The :c:member:`~PyTypeObject.tp_getattr` handler is called when the object requires an attribute look-up. It is called in the same situations where the :meth:`__getattr__` method of a class would be called. @@ -1265,7 +1265,7 @@ A likely way to handle this is (1) to implement a set of functions (such as :c:func:`newdatatype_getSize` and :c:func:`newdatatype_setSize` in the example below), (2) provide a method table listing these functions, and (3) provide a getattr function that returns the result of a lookup in that table. The method -table uses the same structure as the :attr:`tp_methods` field of the type +table uses the same structure as the :c:member:`~PyTypeObject.tp_methods` field of the type object. Here is an example:: @@ -1284,11 +1284,11 @@ Here is an example:: return Py_FindMethod(newdatatype_methods, (PyObject *)obj, name); } -The :attr:`tp_setattr` handler is called when the :meth:`__setattr__` or +The :c:member:`~PyTypeObject.tp_setattr` handler is called when the :meth:`__setattr__` or :meth:`__delattr__` method of a class instance would be called. When an attribute should be deleted, the third parameter will be *NULL*. Here is an example that simply raises an exception; if this were really all you wanted, the -:attr:`tp_setattr` handler should be set to *NULL*. :: +:c:member:`~PyTypeObject.tp_setattr` handler should be set to *NULL*. :: static int newdatatype_setattr(newdatatypeobject *obj, char *name, PyObject *v) @@ -1305,7 +1305,7 @@ Object Comparison cmpfunc tp_compare; -The :attr:`tp_compare` handler is called when comparisons are needed and the +The :c:member:`~PyTypeObject.tp_compare` handler is called when comparisons are needed and the object does not implement the specific rich comparison method which matches the requested comparison. (It is always used if defined and the :c:func:`PyObject_Compare` or :c:func:`PyObject_Cmp` functions are used, or if @@ -1316,7 +1316,7 @@ allowed to return arbitrary negative or positive integers for less than and greater than, respectively; as of Python 2.2, this is no longer allowed. In the future, other return values may be assigned a different meaning.) -A :attr:`tp_compare` handler may raise an exception. In this case it should +A :c:member:`~PyTypeObject.tp_compare` handler may raise an exception. In this case it should return a negative value. The caller has to test for the exception using :c:func:`PyErr_Occurred`. @@ -1391,7 +1391,7 @@ instance of your data type. Here is a moderately pointless example:: This function is called when an instance of your data type is "called", for example, if ``obj1`` is an instance of your data type and the Python script -contains ``obj1('hello')``, the :attr:`tp_call` handler is invoked. +contains ``obj1('hello')``, the :c:member:`~PyTypeObject.tp_call` handler is invoked. This function takes three arguments: @@ -1480,7 +1480,7 @@ those objects which do not benefit by weak referencing (such as numbers). For an object to be weakly referencable, the extension must include a :c:type:`PyObject\*` field in the instance structure for the use of the weak reference mechanism; it must be initialized to *NULL* by the object's -constructor. It must also set the :attr:`tp_weaklistoffset` field of the +constructor. It must also set the :c:member:`~PyTypeObject.tp_weaklistoffset` field of the corresponding type object to the offset of the field. For example, the instance type is defined with the following structure:: @@ -1566,7 +1566,7 @@ might be something like the following:: .. [#] This is true when we know that the object is a basic type, like a string or a float. -.. [#] We relied on this in the :attr:`tp_dealloc` handler in this example, because our +.. [#] We relied on this in the :c:member:`~PyTypeObject.tp_dealloc` handler in this example, because our type doesn't support garbage collection. Even if a type supports garbage collection, there are calls that can be made to "untrack" the object from garbage collection, however, these calls are advanced and not covered here. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 80a2d925ceb..f02f09d7d04 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -132,8 +132,8 @@ The :mod:`gc` module provides the following functions: Return a list of objects directly referred to by any of the arguments. The referents returned are those objects visited by the arguments' C-level - :attr:`tp_traverse` methods (if any), and may not be all objects actually - directly reachable. :attr:`tp_traverse` methods are supported only by objects + :c:member:`~PyTypeObject.tp_traverse` methods (if any), and may not be all objects actually + directly reachable. :c:member:`~PyTypeObject.tp_traverse` methods are supported only by objects that support garbage collection, and are only required to visit objects that may be involved in a cycle. So, for example, if an integer is directly reachable from an argument, that integer object may or may not appear in the result list. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 0001453e5d0..756339dfad7 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -614,7 +614,7 @@ support: iterators for those iteration types. (An example of an object supporting multiple forms of iteration would be a tree structure which supports both breadth-first and depth-first traversal.) This method corresponds to the - :attr:`tp_iter` slot of the type structure for Python objects in the Python/C + :c:member:`~PyTypeObject.tp_iter` slot of the type structure for Python objects in the Python/C API. The iterator objects themselves are required to support the following two @@ -625,7 +625,7 @@ methods, which together form the :dfn:`iterator protocol`: Return the iterator object itself. This is required to allow both containers and iterators to be used with the :keyword:`for` and :keyword:`in` statements. - This method corresponds to the :attr:`tp_iter` slot of the type structure for + This method corresponds to the :c:member:`~PyTypeObject.tp_iter` slot of the type structure for Python objects in the Python/C API. @@ -633,7 +633,7 @@ methods, which together form the :dfn:`iterator protocol`: Return the next item from the container. If there are no further items, raise the :exc:`StopIteration` exception. This method corresponds to the - :attr:`tp_iternext` slot of the type structure for Python objects in the + :c:member:`~PyTypeObject.tp_iternext` slot of the type structure for Python objects in the Python/C API. Python defines several iterator objects to support iteration over general and diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 412c1d0435b..5f6fe11d778 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -450,9 +450,9 @@ signal that the iterator is done. Python classes can define an :meth:`__iter__` method, which should create and return a new iterator for the object; if the object is its own iterator, this method can just return ``self``. In particular, iterators will usually be their -own iterators. Extension types implemented in C can implement a :attr:`tp_iter` +own iterators. Extension types implemented in C can implement a :c:member:`~PyTypeObject.tp_iter` function in order to return an iterator, and extension types that want to behave -as iterators can define a :attr:`tp_iternext` function. +as iterators can define a :c:member:`~PyTypeObject.tp_iternext` function. So, after all this, what do iterators actually do? They have one required method, :meth:`next`, which takes no arguments and returns the next value. When @@ -478,7 +478,7 @@ there are no more values to be returned, calling :meth:`next` should raise the In 2.2, Python's :keyword:`for` statement no longer expects a sequence; it expects something for which :func:`iter` will return an iterator. For backward compatibility and convenience, an iterator is automatically constructed for -sequences that don't implement :meth:`__iter__` or a :attr:`tp_iter` slot, so +sequences that don't implement :meth:`__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so ``for i in [1,2,3]`` will still work. Wherever the Python interpreter loops over a sequence, it's been changed to use the iterator protocol. This means you can do things like this:: From 6a07df6da71c899a99a8c161d6825dceefdf04e6 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 1 Aug 2013 22:25:12 +0200 Subject: [PATCH 257/480] Fix tkinter regression introduced by the security fix in #16248. --- Lib/lib-tk/Tkinter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/lib-tk/Tkinter.py b/Lib/lib-tk/Tkinter.py index fcb320f4418..6947bcc41bc 100644 --- a/Lib/lib-tk/Tkinter.py +++ b/Lib/lib-tk/Tkinter.py @@ -1736,7 +1736,7 @@ def __init__(self, screenName=None, baseName=None, className='Tk', # ensure that self.tk is always _something_. self.tk = None if baseName is None: - import sys, os + import os baseName = os.path.basename(sys.argv[0]) baseName, ext = os.path.splitext(baseName) if ext not in ('.py', '.pyc', '.pyo'): From 3f16cd439da0e201788cfd6865a2551b91271bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Thu, 1 Aug 2013 23:55:26 +0200 Subject: [PATCH 258/480] Add 2.7.5 uuid --- Tools/msi/uuids.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/msi/uuids.py b/Tools/msi/uuids.py index cf5bfd239f6..a4b3ded74d0 100644 --- a/Tools/msi/uuids.py +++ b/Tools/msi/uuids.py @@ -60,4 +60,5 @@ '2.7.3150':'{C0C31BCC-56FB-42a7-8766-D29E1BD74C7C}', # 2.7.3 '2.7.4121':'{47F45F45-72D7-4e54-AF41-26767EDE95CF}', # 2.7.4rc1 '2.7.4150':'{84ADC96C-B7E0-4938-9D6E-2B640D5DA224}', # 2.7.4 + '2.7.5150':'{DBDD570E-0952-475f-9453-AB88F3DD5659}', # 2.7.5 } From 0a8287b6315497919817911c43448fc12b10e396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Thu, 1 Aug 2013 23:57:21 +0200 Subject: [PATCH 259/480] Add 2.7.6 uuids. --- Tools/msi/uuids.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/msi/uuids.py b/Tools/msi/uuids.py index a4b3ded74d0..7c54e513dec 100644 --- a/Tools/msi/uuids.py +++ b/Tools/msi/uuids.py @@ -61,4 +61,6 @@ '2.7.4121':'{47F45F45-72D7-4e54-AF41-26767EDE95CF}', # 2.7.4rc1 '2.7.4150':'{84ADC96C-B7E0-4938-9D6E-2B640D5DA224}', # 2.7.4 '2.7.5150':'{DBDD570E-0952-475f-9453-AB88F3DD5659}', # 2.7.5 + '2.7.6121':'{D1EBC07F-A7B1-4163-83DB-AE813CEF392F}', # 2.7.6rc1 + '2.7.6150':'{C3CC4DF5-39A5-4027-B136-2B3E1F5AB6E2}', # 2.7.6 } From 031cac9e7fd7d3ce769c74d2535be2f3022e1293 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Thu, 1 Aug 2013 21:19:09 -0700 Subject: [PATCH 260/480] Issue #17557: Fix os.getgroups() to work with the modified behavior of getgroups(2) on OS X 10.8. Original patch by Mateusz Lenik. --- Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/posixmodule.c | 30 ++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/Misc/ACKS b/Misc/ACKS index cf553b70e6f..822a523d05c 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -599,6 +599,7 @@ Petri Lehtinen Luke Kenneth Casson Leighton Tshepang Lekhonkhobe Marc-André Lemburg +Mateusz Lenik John Lenton Kostyantyn Leschenko Christopher Tur Lesniewski-Laas diff --git a/Misc/NEWS b/Misc/NEWS index ee5c8f044fd..e3394c47fec 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -26,6 +26,9 @@ Core and Builtins Library ------- +- Issue #17557: Fix os.getgroups() to work with the modified behavior of + getgroups(2) on OS X 10.8. Original patch by Mateusz Lenik. + - Issue #18455: multiprocessing should not retry connect() with same socket. - Issue #18513: Fix behaviour of cmath.rect w.r.t. signed zeros on OS X 10.8 + diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 65ade9fb9fd..8352dd3e00a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4054,6 +4054,34 @@ posix_getgroups(PyObject *self, PyObject *noargs) gid_t* alt_grouplist = grouplist; int n; +#ifdef __APPLE__ + /* Issue #17557: As of OS X 10.8, getgroups(2) no longer raises EINVAL if + * there are more groups than can fit in grouplist. Therefore, on OS X + * always first call getgroups with length 0 to get the actual number + * of groups. + */ + n = getgroups(0, NULL); + if (n < 0) { + return posix_error(); + } else if (n <= MAX_GROUPS) { + /* groups will fit in existing array */ + alt_grouplist = grouplist; + } else { + alt_grouplist = PyMem_Malloc(n * sizeof(gid_t)); + if (alt_grouplist == NULL) { + errno = EINVAL; + return posix_error(); + } + } + + n = getgroups(n, alt_grouplist); + if (n == -1) { + if (alt_grouplist != grouplist) { + PyMem_Free(alt_grouplist); + } + return posix_error(); + } +#else n = getgroups(MAX_GROUPS, grouplist); if (n < 0) { if (errno == EINVAL) { @@ -4080,6 +4108,8 @@ posix_getgroups(PyObject *self, PyObject *noargs) return posix_error(); } } +#endif + result = PyList_New(n); if (result != NULL) { int i; From b7c605e3312d597ff1b8c73ceeca427c55fe316f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Fri, 2 Aug 2013 10:01:46 +0200 Subject: [PATCH 261/480] Issue #18325: Fix a test_kqueue failure on OpenBSD: kevent's data and event members are integers. --- Lib/test/test_kqueue.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py index 2f8ff2f382d..3dffc73ce6a 100644 --- a/Lib/test/test_kqueue.py +++ b/Lib/test/test_kqueue.py @@ -70,13 +70,13 @@ def test_create_event(self): self.assertEqual(ev, ev) self.assertNotEqual(ev, other) - bignum = sys.maxsize * 2 + 1 - ev = select.kevent(bignum, 1, 2, 3, sys.maxsize, bignum) + bignum = 0x7fff + ev = select.kevent(bignum, 1, 2, 3, bignum - 1, bignum) self.assertEqual(ev.ident, bignum) self.assertEqual(ev.filter, 1) self.assertEqual(ev.flags, 2) self.assertEqual(ev.fflags, 3) - self.assertEqual(ev.data, sys.maxsize) + self.assertEqual(ev.data, bignum - 1) self.assertEqual(ev.udata, bignum) self.assertEqual(ev, ev) self.assertNotEqual(ev, other) From d4c0c561c4f407be1d55a80100c2082b6e499bde Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 3 Aug 2013 19:26:33 +0300 Subject: [PATCH 262/480] Issue #17998: Fix an internal error in regular expression engine. --- Lib/test/test_re.py | 10 ++++++++++ Misc/NEWS | 2 ++ Modules/_sre.c | 12 ++++++------ Modules/sre.h | 4 ++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 18a81a21418..2be5f5cb2ea 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -897,6 +897,16 @@ def test_group_name_in_exception(self): with self.assertRaisesRegexp(sre_constants.error, '\?foo'): re.compile('(?P)') + def test_issue17998(self): + for reps in '*', '+', '?', '{1}': + for mod in '', '?': + pattern = '.' + reps + mod + 'yz' + self.assertEqual(re.compile(pattern, re.S).findall('xyz'), + ['xyz'], msg=pattern) + pattern = pattern.encode() + self.assertEqual(re.compile(pattern, re.S).findall(b'xyz'), + [b'xyz'], msg=pattern) + def run_re_tests(): from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR diff --git a/Misc/NEWS b/Misc/NEWS index e3394c47fec..ef614c7ce6b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -26,6 +26,8 @@ Core and Builtins Library ------- +- Issue #17998: Fix an internal error in regular expression engine. + - Issue #17557: Fix os.getgroups() to work with the modified behavior of getgroups(2) on OS X 10.8. Original patch by Mateusz Lenik. diff --git a/Modules/_sre.c b/Modules/_sre.c index 478416fb1da..c9e4f7355e9 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -1028,7 +1028,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) TRACE(("|%p|%p|REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1], ctx->pattern[2])); - if (ctx->pattern[1] > end - ctx->ptr) + if ((Py_ssize_t) ctx->pattern[1] > end - ctx->ptr) RETURN_FAILURE; /* cannot match */ state->ptr = ctx->ptr; @@ -1111,7 +1111,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) TRACE(("|%p|%p|MIN_REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1], ctx->pattern[2])); - if (ctx->pattern[1] > end - ctx->ptr) + if ((Py_ssize_t) ctx->pattern[1] > end - ctx->ptr) RETURN_FAILURE; /* cannot match */ state->ptr = ctx->ptr; @@ -1210,7 +1210,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) TRACE(("|%p|%p|MAX_UNTIL %d\n", ctx->pattern, ctx->ptr, ctx->count)); - if (ctx->count < ctx->u.rep->pattern[1]) { + if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { /* not enough matches */ ctx->u.rep->count = ctx->count; DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1, @@ -1224,7 +1224,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) RETURN_FAILURE; } - if ((ctx->count < ctx->u.rep->pattern[2] || + if ((ctx->count < (Py_ssize_t) ctx->u.rep->pattern[2] || ctx->u.rep->pattern[2] == SRE_MAXREPEAT) && state->ptr != ctx->u.rep->last_ptr) { /* we may have enough matches, but if we can @@ -1273,7 +1273,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) TRACE(("|%p|%p|MIN_UNTIL %d %p\n", ctx->pattern, ctx->ptr, ctx->count, ctx->u.rep->pattern)); - if (ctx->count < ctx->u.rep->pattern[1]) { + if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { /* not enough matches */ ctx->u.rep->count = ctx->count; DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1, @@ -1302,7 +1302,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) LASTMARK_RESTORE(); - if ((ctx->count >= ctx->u.rep->pattern[2] + if ((ctx->count >= (Py_ssize_t) ctx->u.rep->pattern[2] && ctx->u.rep->pattern[2] != SRE_MAXREPEAT) || state->ptr == ctx->u.rep->last_ptr) RETURN_FAILURE; diff --git a/Modules/sre.h b/Modules/sre.h index 15ed3d511b7..22a2d5dd96c 100644 --- a/Modules/sre.h +++ b/Modules/sre.h @@ -20,14 +20,14 @@ # if SIZEOF_SIZE_T > 4 # define SRE_MAXREPEAT (~(SRE_CODE)0) # else -# define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX + 1u) +# define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX) # endif #else # define SRE_CODE unsigned int # if SIZEOF_SIZE_T > SIZEOF_INT # define SRE_MAXREPEAT (~(SRE_CODE)0) # else -# define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX + 1u) +# define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX) # endif #endif From 9c1639aae19a05cee2701251d5a54b720cb4b7e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sat, 3 Aug 2013 20:24:00 +0200 Subject: [PATCH 263/480] Issue #16067: Add description into MSI file to replace installer's temporary name. --- Misc/NEWS | 2 ++ Tools/msi/msi.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index ef614c7ce6b..64668dd4ea7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -98,6 +98,8 @@ Tools/Demos Build ----- +- Issue #16067: Add description into MSI file to replace installer's temporary name. + - Issue #18256: Compilation fix for recent AIX releases. Patch by David Edelsohn. diff --git a/Tools/msi/msi.py b/Tools/msi/msi.py index d2caf345b1d..2f0d9637140 100644 --- a/Tools/msi/msi.py +++ b/Tools/msi/msi.py @@ -1392,7 +1392,10 @@ def merge(msi, feature, rootdir, modules): # certname (from config.py) should be (a substring of) # the certificate subject, e.g. "Python Software Foundation" if certname: - os.system('signtool sign /n "%s" /t http://timestamp.verisign.com/scripts/timestamp.dll %s' % (certname, msiname)) + os.system('signtool sign /n "%s" ' + '/t http://timestamp.verisign.com/scripts/timestamp.dll ' + '/d "Python %s" ' + '%s' % (certname, full_current_version, msiname)) if pdbzip: build_pdbzip() From d30106dddd09af83ca5f0c49ac5a499f891dbdbe Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 4 Aug 2013 13:22:30 +0300 Subject: [PATCH 264/480] Issue #18647: Temporary disable the "nothing to repeat" check to make buildbots happy. --- Lib/sre_compile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py index 7cda2b657b8..425a1f80cb2 100644 --- a/Lib/sre_compile.py +++ b/Lib/sre_compile.py @@ -355,8 +355,8 @@ def _optimize_unicode(charset, fixup): def _simple(av): # check if av is a "simple" operator lo, hi = av[2].getwidth() - if lo == 0 and hi == MAXREPEAT: - raise error, "nothing to repeat" + #if lo == 0 and hi == MAXREPEAT: + # raise error, "nothing to repeat" return lo == hi == 1 and av[2][0][0] != SUBPATTERN def _compile_info(code, pattern, flags): From b9b7349a283f1c582b2a466539129df9e5e38ff9 Mon Sep 17 00:00:00 2001 From: Eli Bendersky Date: Sun, 4 Aug 2013 06:09:49 -0700 Subject: [PATCH 265/480] Issue #13612: Fix a buffer overflow in case of a multi-byte encoding. This is a belated backport of f7b47fb30169; Patch by Serhiy Storchaka. --- Lib/test/test_xml_etree.py | 6 ++++++ Modules/_elementtree.c | 2 ++ Modules/pyexpat.c | 7 +++++++ 3 files changed, 15 insertions(+) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 4f06d20c045..adb785247d5 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -883,6 +883,12 @@ def check_encoding(encoding): >>> check_encoding("iso-8859-15") >>> check_encoding("cp437") >>> check_encoding("mac-roman") + >>> check_encoding("gbk") + Traceback (most recent call last): + ValueError: multi-byte encodings are not supported + >>> check_encoding("cp037") + Traceback (most recent call last): + ParseError: unknown encoding: line 1, column 30 """ ET.XML("" % encoding) diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 379aa01d0a5..b9abcac8d0c 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2427,6 +2427,8 @@ expat_unknown_encoding_handler(XMLParserObject *self, const XML_Char *name, if (PyUnicode_GET_SIZE(u) != 256) { Py_DECREF(u); + PyErr_SetString(PyExc_ValueError, + "multi-byte encodings are not supported"); return XML_STATUS_ERROR; } diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index f2691136ca8..8de3fb17d33 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1252,6 +1252,13 @@ PyUnknownEncodingHandler(void *encodingHandlerData, if (_u_string == NULL) return result; + if (PyUnicode_GET_SIZE(_u_string) != 256) { + Py_DECREF(_u_string); + PyErr_SetString(PyExc_ValueError, + "multi-byte encodings are not supported"); + return result; + } + for (i = 0; i < 256; i++) { /* Stupid to access directly, but fast */ Py_UNICODE c = _u_string->str[i]; From 0cd5dee940175d5323846bf5d89e902b3c596432 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 4 Aug 2013 15:39:56 -0400 Subject: [PATCH 266/480] Issue #18151: Replace remaining Idle 'open...close' pairs with 'with open'. --- Lib/idlelib/EditorWindow.py | 5 +---- Lib/idlelib/IOBinding.py | 11 ++++------- Lib/idlelib/ScriptBinding.py | 5 ++--- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/Lib/idlelib/EditorWindow.py b/Lib/idlelib/EditorWindow.py index 5de53a99d94..07ca5562330 100644 --- a/Lib/idlelib/EditorWindow.py +++ b/Lib/idlelib/EditorWindow.py @@ -894,11 +894,8 @@ def update_recent_files_list(self, new_file=None): "Load and update the recent files list and menus" rf_list = [] if os.path.exists(self.recent_files_path): - rf_list_file = open(self.recent_files_path,'r') - try: + with open(self.recent_files_path, 'r') as rf_list_file: rf_list = rf_list_file.readlines() - finally: - rf_list_file.close() if new_file: new_file = os.path.abspath(new_file) + '\n' if new_file in rf_list: diff --git a/Lib/idlelib/IOBinding.py b/Lib/idlelib/IOBinding.py index f0d4d227c3c..ba45ee833fe 100644 --- a/Lib/idlelib/IOBinding.py +++ b/Lib/idlelib/IOBinding.py @@ -248,9 +248,8 @@ def loadfile(self, filename): try: # open the file in binary mode so that we can handle # end-of-line convention ourselves. - f = open(filename,'rb') - chars = f.read() - f.close() + with open(filename, 'rb') as f: + chars = f.read() except IOError as msg: tkMessageBox.showerror("I/O Error", str(msg), master=self.text) return False @@ -383,10 +382,8 @@ def writefile(self, filename): if self.eol_convention != "\n": chars = chars.replace("\n", self.eol_convention) try: - f = open(filename, "wb") - f.write(chars) - f.flush() - f.close() + with open(filename, "wb") as f: + f.write(chars) return True except IOError as msg: tkMessageBox.showerror("I/O Error", str(msg), diff --git a/Lib/idlelib/ScriptBinding.py b/Lib/idlelib/ScriptBinding.py index 665b3b241cc..a024ebfb58e 100644 --- a/Lib/idlelib/ScriptBinding.py +++ b/Lib/idlelib/ScriptBinding.py @@ -87,9 +87,8 @@ def checksyntax(self, filename): self.shell = shell = self.flist.open_shell() saved_stream = shell.get_warning_stream() shell.set_warning_stream(shell.stderr) - f = open(filename, 'r') - source = f.read() - f.close() + with open(filename, 'r') as f: + source = f.read() if '\r' in source: source = re.sub(r"\r\n", "\n", source) source = re.sub(r"\r", "\n", source) From c5d08f86173318408290c16c24e7880590d218f2 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 4 Aug 2013 22:35:37 -0700 Subject: [PATCH 267/480] Silence compiler warning for unused declaration. --- Objects/setobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index af1ce16bc29..db5cee28403 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -212,7 +212,6 @@ static int set_insert_key(register PySetObject *so, PyObject *key, long hash) { register setentry *entry; - typedef setentry *(*lookupfunc)(PySetObject *, PyObject *, long); assert(so->lookup != NULL); entry = so->lookup(so, key, hash); From 1c226e35ef57333c95492204a6e9229636e4947f Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Mon, 5 Aug 2013 17:56:17 +0100 Subject: [PATCH 268/480] Issue #18661: typo in grp.struct_group docstring. --- Modules/grpmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 55da9b8aa28..040d8b0b100 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -11,7 +11,7 @@ static PyStructSequence_Field struct_group_type_fields[] = { {"gr_name", "group name"}, {"gr_passwd", "password"}, {"gr_gid", "group id"}, - {"gr_mem", "group memebers"}, + {"gr_mem", "group members"}, {0} }; From 29e705b012fdd582f0d1aede1950af40d3a62be9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 6 Aug 2013 16:56:26 +0300 Subject: [PATCH 269/480] Issue #15866: The xmlcharrefreplace error handler no more produces two XML entities for a non-BMP character on narrow build. --- Lib/test/test_codeccallbacks.py | 25 ++++++++-- Lib/test/test_unicode.py | 12 +++++ Misc/NEWS | 3 ++ Modules/_testcapimodule.c | 2 +- Objects/unicodeobject.c | 82 ++++++++++++++++++++++++--------- Python/codecs.c | 66 ++++++++++++++------------ 6 files changed, 135 insertions(+), 55 deletions(-) diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 61c2df20c43..ecaf9970a1d 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -66,15 +66,34 @@ def test_xmlcharrefreplace(self): # replace unencodable characters which numeric character entities. # For ascii, latin-1 and charmaps this is completely implemented # in C and should be reasonably fast. - s = u"\u30b9\u30d1\u30e2 \xe4nd eggs" + s = u"\u30b9\u30d1\u30e2 \xe4nd egg\u0161" self.assertEqual( s.encode("ascii", "xmlcharrefreplace"), - "スパモ änd eggs" + "スパモ änd eggš" ) self.assertEqual( s.encode("latin-1", "xmlcharrefreplace"), - "スパモ \xe4nd eggs" + "スパモ \xe4nd eggš" ) + self.assertEqual( + s.encode("iso-8859-15", "xmlcharrefreplace"), + "スパモ \xe4nd egg\xa8" + ) + + def test_xmlcharrefreplace_with_surrogates(self): + tests = [(u'\U0001f49d', '💝'), + (u'\ud83d', '�'), + (u'\udc9d', '�'), + (u'\ud83d\udc9d', '💝' if len(u'\U0001f49d') > 1 else + '��'), + ] + for encoding in ['ascii', 'latin1', 'iso-8859-15']: + for s, exp in tests: + self.assertEqual(s.encode(encoding, 'xmlcharrefreplace'), + exp, msg='%r.encode(%r)' % (s, encoding)) + self.assertEqual((s+'X').encode(encoding, 'xmlcharrefreplace'), + exp+'X', + msg='%r.encode(%r)' % (s + 'X', encoding)) def test_xmlcharnamereplace(self): # This time use a named character entity for unencodable diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index e44fe03d240..666cab87255 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1658,6 +1658,18 @@ def test_encode_decimal(self): self.assertEqual(unicode_encodedecimal(u"123\u20ac\u0660", "replace"), b'123?0') + def test_encode_decimal_with_surrogates(self): + from _testcapi import unicode_encodedecimal + tests = [(u'\U0001f49d', '💝'), + (u'\ud83d', '�'), + (u'\udc9d', '�'), + (u'\ud83d\udc9d', '💝' if len(u'\U0001f49d') > 1 else + '��'), + ] + for s, exp in tests: + self.assertEqual( + unicode_encodedecimal(u"123" + s, "xmlcharrefreplace"), + '123' + exp) def test_main(): test_support.run_unittest(__name__) diff --git a/Misc/NEWS b/Misc/NEWS index 64668dd4ea7..af3a94bc78a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -9,6 +9,9 @@ What's New in Python 2.7.6? Core and Builtins ----------------- +- Issue #15866: The xmlcharrefreplace error handler no more produces two XML + entities for a non-BMP character on narrow build. + - Issue #18184: PyUnicode_FromFormat() and PyUnicode_FromFormatV() now raise OverflowError when an argument of %c format is out of range. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b0386f0cace..4e7d47d8593 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1118,7 +1118,7 @@ unicode_encodedecimal(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "u#|s", &unicode, &length, &errors)) return NULL; - decimal_length = length * 7; /* len('€') */ + decimal_length = length * 10; /* len('􏿿') */ decimal = PyBytes_FromStringAndSize(NULL, decimal_length); if (decimal == NULL) return NULL; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 64a5ef557c0..866eb9b0589 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -547,6 +547,37 @@ PyObject *PyUnicode_FromString(const char *u) return PyUnicode_FromStringAndSize(u, size); } +/* _Py_UNICODE_NEXT is a private macro used to retrieve the character pointed + * by 'ptr', possibly combining surrogate pairs on narrow builds. + * 'ptr' and 'end' must be Py_UNICODE*, with 'ptr' pointing at the character + * that should be returned and 'end' pointing to the end of the buffer. + * ('end' is used on narrow builds to detect a lone surrogate at the + * end of the buffer that should be returned unchanged.) + * The ptr and end arguments should be side-effect free and ptr must an lvalue. + * The type of the returned char is always Py_UCS4. + * + * Note: the macro advances ptr to next char, so it might have side-effects + * (especially if used with other macros). + */ + +/* helper macros used by _Py_UNICODE_NEXT */ +#define _Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDBFF) +#define _Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= ch && ch <= 0xDFFF) +/* Join two surrogate characters and return a single Py_UCS4 value. */ +#define _Py_UNICODE_JOIN_SURROGATES(high, low) \ + (((((Py_UCS4)(high) & 0x03FF) << 10) | \ + ((Py_UCS4)(low) & 0x03FF)) + 0x10000) + +#ifdef Py_UNICODE_WIDE +#define _Py_UNICODE_NEXT(ptr, end) *(ptr)++ +#else +#define _Py_UNICODE_NEXT(ptr, end) \ + (((_Py_UNICODE_IS_HIGH_SURROGATE(*(ptr)) && (ptr) < (end)) && \ + _Py_UNICODE_IS_LOW_SURROGATE((ptr)[1])) ? \ + ((ptr) += 2,_Py_UNICODE_JOIN_SURROGATES((ptr)[-2], (ptr)[-1])) : \ + (Py_UCS4)*(ptr)++) +#endif + #ifdef HAVE_WCHAR_H #if (Py_UNICODE_SIZE == 2) && defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) @@ -3642,26 +3673,22 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, case 4: /* xmlcharrefreplace */ respos = str-PyString_AS_STRING(res); /* determine replacement size (temporarily (mis)uses p) */ - for (p = collstart, repsize = 0; p < collend; ++p) { - if (*p<10) + for (p = collstart, repsize = 0; p < collend;) { + Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend); + if (ch < 10) repsize += 2+1+1; - else if (*p<100) + else if (ch < 100) repsize += 2+2+1; - else if (*p<1000) + else if (ch < 1000) repsize += 2+3+1; - else if (*p<10000) + else if (ch < 10000) repsize += 2+4+1; -#ifndef Py_UNICODE_WIDE - else + else if (ch < 100000) repsize += 2+5+1; -#else - else if (*p<100000) - repsize += 2+5+1; - else if (*p<1000000) + else if (ch < 1000000) repsize += 2+6+1; else repsize += 2+7+1; -#endif } requiredsize = respos+repsize+(endp-collend); if (requiredsize > ressize) { @@ -3673,8 +3700,9 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, ressize = requiredsize; } /* generate replacement (temporarily (mis)uses p) */ - for (p = collstart; p < collend; ++p) { - str += sprintf(str, "&#%d;", (int)*p); + for (p = collstart; p < collend;) { + Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend); + str += sprintf(str, "&#%d;", (int)ch); } p = collend; break; @@ -4649,11 +4677,20 @@ int charmap_encoding_error( *inpos = collendpos; break; case 4: /* xmlcharrefreplace */ - /* generate replacement (temporarily (mis)uses p) */ - for (collpos = collstartpos; collpos < collendpos; ++collpos) { + /* generate replacement */ + for (collpos = collstartpos; collpos < collendpos;) { char buffer[2+29+1+1]; char *cp; - sprintf(buffer, "&#%d;", (int)p[collpos]); + Py_UCS4 ch = p[collpos++]; +#ifndef Py_UNICODE_WIDE + if ((0xD800 <= ch && ch <= 0xDBFF) && + (collpos < collendpos) && + (0xDC00 <= p[collpos] && p[collpos] <= 0xDFFF)) { + ch = ((((ch & 0x03FF) << 10) | + ((Py_UCS4)p[collpos++] & 0x03FF)) + 0x10000); + } +#endif + sprintf(buffer, "&#%d;", (int)ch); for (cp = buffer; *cp; ++cp) { x = charmapencode_output(*cp, mapping, res, respos); if (x==enc_EXCEPTION) @@ -5068,10 +5105,11 @@ PyObject *PyUnicode_TranslateCharmap(const Py_UNICODE *p, break; case 4: /* xmlcharrefreplace */ /* generate replacement (temporarily (mis)uses p) */ - for (p = collstart; p < collend; ++p) { + for (p = collstart; p < collend;) { char buffer[2+29+1+1]; char *cp; - sprintf(buffer, "&#%d;", (int)*p); + Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend); + sprintf(buffer, "&#%d;", (int)ch); if (charmaptranslate_makespace(&res, &str, (str-PyUnicode_AS_UNICODE(res))+strlen(buffer)+(endp-collend))) goto onError; @@ -5222,8 +5260,10 @@ int PyUnicode_EncodeDecimal(Py_UNICODE *s, break; case 4: /* xmlcharrefreplace */ /* generate replacement (temporarily (mis)uses p) */ - for (p = collstart; p < collend; ++p) - output += sprintf(output, "&#%d;", (int)*p); + for (p = collstart; p < collend;) { + Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend); + output += sprintf(output, "&#%d;", ch); + } p = collend; break; default: diff --git a/Python/codecs.c b/Python/codecs.c index 7334eb3e360..91147a07a32 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -556,6 +556,7 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) PyObject *res; Py_UNICODE *p; Py_UNICODE *startp; + Py_UNICODE *e; Py_UNICODE *outp; int ressize; if (PyUnicodeEncodeError_GetStart(exc, &start)) @@ -565,26 +566,31 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) if (!(object = PyUnicodeEncodeError_GetObject(exc))) return NULL; startp = PyUnicode_AS_UNICODE(object); - for (p = startp+start, ressize = 0; p < startp+end; ++p) { - if (*p<10) + e = startp + end; + for (p = startp+start, ressize = 0; p < e;) { + Py_UCS4 ch = *p++; +#ifndef Py_UNICODE_WIDE + if ((0xD800 <= ch && ch <= 0xDBFF) && + (p < e) && + (0xDC00 <= *p && *p <= 0xDFFF)) { + ch = ((((ch & 0x03FF) << 10) | + ((Py_UCS4)*p++ & 0x03FF)) + 0x10000); + } +#endif + if (ch < 10) ressize += 2+1+1; - else if (*p<100) + else if (ch < 100) ressize += 2+2+1; - else if (*p<1000) + else if (ch < 1000) ressize += 2+3+1; - else if (*p<10000) + else if (ch < 10000) ressize += 2+4+1; -#ifndef Py_UNICODE_WIDE - else - ressize += 2+5+1; -#else - else if (*p<100000) + else if (ch < 100000) ressize += 2+5+1; - else if (*p<1000000) + else if (ch < 1000000) ressize += 2+6+1; else ressize += 2+7+1; -#endif } /* allocate replacement */ res = PyUnicode_FromUnicode(NULL, ressize); @@ -593,40 +599,41 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) return NULL; } /* generate replacement */ - for (p = startp+start, outp = PyUnicode_AS_UNICODE(res); - p < startp+end; ++p) { - Py_UNICODE c = *p; + for (p = startp+start, outp = PyUnicode_AS_UNICODE(res); p < e;) { int digits; int base; + Py_UCS4 ch = *p++; +#ifndef Py_UNICODE_WIDE + if ((0xD800 <= ch && ch <= 0xDBFF) && + (p < startp+end) && + (0xDC00 <= *p && *p <= 0xDFFF)) { + ch = ((((ch & 0x03FF) << 10) | + ((Py_UCS4)*p++ & 0x03FF)) + 0x10000); + } +#endif *outp++ = '&'; *outp++ = '#'; - if (*p<10) { + if (ch < 10) { digits = 1; base = 1; } - else if (*p<100) { + else if (ch < 100) { digits = 2; base = 10; } - else if (*p<1000) { + else if (ch < 1000) { digits = 3; base = 100; } - else if (*p<10000) { + else if (ch < 10000) { digits = 4; base = 1000; } -#ifndef Py_UNICODE_WIDE - else { - digits = 5; - base = 10000; - } -#else - else if (*p<100000) { + else if (ch < 100000) { digits = 5; base = 10000; } - else if (*p<1000000) { + else if (ch < 1000000) { digits = 6; base = 100000; } @@ -634,10 +641,9 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) digits = 7; base = 1000000; } -#endif while (digits-->0) { - *outp++ = '0' + c/base; - c %= base; + *outp++ = '0' + ch/base; + ch %= base; base /= 10; } *outp++ = ';'; From 79c7dda1530ec811bf9aba78ac6cdcd8e6c2f06e Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Thu, 8 Aug 2013 20:09:19 +0300 Subject: [PATCH 270/480] #18357: add tests for dictview set difference. Patch by Fraser Tweedale. --- Lib/test/test_dictviews.py | 15 +++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 3 files changed, 19 insertions(+) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index b264ff2ba16..30cfb93be7d 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -112,6 +112,13 @@ def test_keys_set_operations(self): self.assertEqual(d1.viewkeys() ^ set(d3.viewkeys()), {'a', 'b', 'd', 'e'}) + self.assertEqual(d1.viewkeys() - d1.viewkeys(), set()) + self.assertEqual(d1.viewkeys() - d2.viewkeys(), {'a'}) + self.assertEqual(d1.viewkeys() - d3.viewkeys(), {'a', 'b'}) + self.assertEqual(d1.viewkeys() - set(d1.viewkeys()), set()) + self.assertEqual(d1.viewkeys() - set(d2.viewkeys()), {'a'}) + self.assertEqual(d1.viewkeys() - set(d3.viewkeys()), {'a', 'b'}) + def test_items_set_operations(self): d1 = {'a': 1, 'b': 2} d2 = {'a': 2, 'b': 2} @@ -144,6 +151,14 @@ def test_items_set_operations(self): self.assertEqual(d1.viewitems() ^ d3.viewitems(), {('a', 1), ('b', 2), ('d', 4), ('e', 5)}) + self.assertEqual(d1.viewitems() - d1.viewitems(), set()) + self.assertEqual(d1.viewitems() - d2.viewitems(), {('a', 1)}) + self.assertEqual(d1.viewitems() - d3.viewitems(), {('a', 1), ('b', 2)}) + self.assertEqual(d1.viewitems() - set(d1.viewitems()), set()) + self.assertEqual(d1.viewitems() - set(d2.viewitems()), {('a', 1)}) + self.assertEqual(d1.viewitems() - set(d3.viewitems()), + {('a', 1), ('b', 2)}) + def test_recursive_repr(self): d = {} d[42] = d.viewvalues() diff --git a/Misc/ACKS b/Misc/ACKS index 822a523d05c..736887d62d8 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1042,6 +1042,7 @@ David Turner Stephen Turner Theodore Turocy Bill Tutt +Fraser Tweedale Doobee R. Tzeck Eren Türkay Lionel Ulmer diff --git a/Misc/NEWS b/Misc/NEWS index af3a94bc78a..e5432b67980 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -142,6 +142,9 @@ IDLE Tests ----- +- Issue #18357: add tests for dictview set difference. + Patch by Fraser Tweedale. + - Issue #11185: Fix test_wait4 under AIX. Patch by Sébastien Sablé. - Issue #18094: test_uuid no more reports skipped tests as passed. From d0077c3663a05a705d2de57a7a696bcbec66eff7 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 10 Aug 2013 16:56:20 -0400 Subject: [PATCH 271/480] Issue #18226: Add docstrings and unittests for idlelib/FormatParagraph.py. Move comment code to a separate function so it can be separately tested. Original patches by Todd Rovito and Phil Webster. --- Lib/idlelib/FormatParagraph.py | 137 ++++--- Lib/idlelib/idle_test/test_formatparagraph.py | 374 ++++++++++++++++++ Misc/NEWS | 3 + 3 files changed, 466 insertions(+), 48 deletions(-) create mode 100644 Lib/idlelib/idle_test/test_formatparagraph.py diff --git a/Lib/idlelib/FormatParagraph.py b/Lib/idlelib/FormatParagraph.py index 557d8a9e0d7..ae4e6e7b911 100644 --- a/Lib/idlelib/FormatParagraph.py +++ b/Lib/idlelib/FormatParagraph.py @@ -1,18 +1,19 @@ -# Extension to format a paragraph - -# Does basic, standard text formatting, and also understands Python -# comment blocks. Thus, for editing Python source code, this -# extension is really only suitable for reformatting these comment -# blocks or triple-quoted strings. - -# Known problems with comment reformatting: -# * If there is a selection marked, and the first line of the -# selection is not complete, the block will probably not be detected -# as comments, and will have the normal "text formatting" rules -# applied. -# * If a comment block has leading whitespace that mixes tabs and -# spaces, they will not be considered part of the same block. -# * Fancy comments, like this bulleted list, arent handled :-) +"""Extension to format a paragraph or selection to a max width. + +Does basic, standard text formatting, and also understands Python +comment blocks. Thus, for editing Python source code, this +extension is really only suitable for reformatting these comment +blocks or triple-quoted strings. + +Known problems with comment reformatting: +* If there is a selection marked, and the first line of the + selection is not complete, the block will probably not be detected + as comments, and will have the normal "text formatting" rules + applied. +* If a comment block has leading whitespace that mixes tabs and + spaces, they will not be considered part of the same block. +* Fancy comments, like this bulleted list, aren't handled :-) +""" import re from idlelib.configHandler import idleConf @@ -32,41 +33,31 @@ def close(self): self.editwin = None def format_paragraph_event(self, event): - maxformatwidth = int(idleConf.GetOption('main','FormatParagraph', - 'paragraph', type='int')) + """Formats paragraph to a max width specified in idleConf. + + If text is selected, format_paragraph_event will start breaking lines + at the max width, starting from the beginning selection. + + If no text is selected, format_paragraph_event uses the current + cursor location to determine the paragraph (lines of text surrounded + by blank lines) and formats it. + """ + maxformatwidth = idleConf.GetOption( + 'main', 'FormatParagraph', 'paragraph', type='int') text = self.editwin.text first, last = self.editwin.get_selection_indices() if first and last: data = text.get(first, last) - comment_header = '' + comment_header = get_comment_header(data) else: first, last, comment_header, data = \ find_paragraph(text, text.index("insert")) if comment_header: - # Reformat the comment lines - convert to text sans header. - lines = data.split("\n") - lines = map(lambda st, l=len(comment_header): st[l:], lines) - data = "\n".join(lines) - # Reformat to maxformatwidth chars or a 20 char width, whichever is greater. - format_width = max(maxformatwidth - len(comment_header), 20) - newdata = reformat_paragraph(data, format_width) - # re-split and re-insert the comment header. - newdata = newdata.split("\n") - # If the block ends in a \n, we dont want the comment - # prefix inserted after it. (Im not sure it makes sense to - # reformat a comment block that isnt made of complete - # lines, but whatever!) Can't think of a clean solution, - # so we hack away - block_suffix = "" - if not newdata[-1]: - block_suffix = "\n" - newdata = newdata[:-1] - builder = lambda item, prefix=comment_header: prefix+item - newdata = '\n'.join(map(builder, newdata)) + block_suffix + newdata = reformat_comment(data, maxformatwidth, comment_header) else: - # Just a normal text format newdata = reformat_paragraph(data, maxformatwidth) text.tag_remove("sel", "1.0", "end") + if newdata != data: text.mark_set("insert", first) text.undo_block_start() @@ -79,31 +70,44 @@ def format_paragraph_event(self, event): return "break" def find_paragraph(text, mark): + """Returns the start/stop indices enclosing the paragraph that mark is in. + + Also returns the comment format string, if any, and paragraph of text + between the start/stop indices. + """ lineno, col = map(int, mark.split(".")) - line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) + line = text.get("%d.0" % lineno, "%d.end" % lineno) + + # Look for start of next paragraph if the index passed in is a blank line while text.compare("%d.0" % lineno, "<", "end") and is_all_white(line): lineno = lineno + 1 - line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) + line = text.get("%d.0" % lineno, "%d.end" % lineno) first_lineno = lineno comment_header = get_comment_header(line) comment_header_len = len(comment_header) + + # Once start line found, search for end of paragraph (a blank line) while get_comment_header(line)==comment_header and \ not is_all_white(line[comment_header_len:]): lineno = lineno + 1 - line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) + line = text.get("%d.0" % lineno, "%d.end" % lineno) last = "%d.0" % lineno - # Search back to beginning of paragraph + + # Search back to beginning of paragraph (first blank line before) lineno = first_lineno - 1 - line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) + line = text.get("%d.0" % lineno, "%d.end" % lineno) while lineno > 0 and \ get_comment_header(line)==comment_header and \ not is_all_white(line[comment_header_len:]): lineno = lineno - 1 - line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) + line = text.get("%d.0" % lineno, "%d.end" % lineno) first = "%d.0" % (lineno+1) + return first, last, comment_header, text.get(first, last) +# This should perhaps be replaced with textwrap.wrap def reformat_paragraph(data, limit): + """Return data reformatted to specified width (limit).""" lines = data.split("\n") i = 0 n = len(lines) @@ -126,7 +130,7 @@ def reformat_paragraph(data, limit): if not word: continue # Can happen when line ends in whitespace if len((partial + word).expandtabs()) > limit and \ - partial != indent1: + partial != indent1: new.append(partial.rstrip()) partial = indent2 partial = partial + word + " " @@ -138,13 +142,50 @@ def reformat_paragraph(data, limit): new.extend(lines[i:]) return "\n".join(new) +def reformat_comment(data, limit, comment_header): + """Return data reformatted to specified width with comment header.""" + + # Remove header from the comment lines + lc = len(comment_header) + data = "\n".join(line[lc:] for line in data.split("\n")) + # Reformat to maxformatwidth chars or a 20 char width, + # whichever is greater. + format_width = max(limit - len(comment_header), 20) + newdata = reformat_paragraph(data, format_width) + # re-split and re-insert the comment header. + newdata = newdata.split("\n") + # If the block ends in a \n, we dont want the comment prefix + # inserted after it. (Im not sure it makes sense to reformat a + # comment block that is not made of complete lines, but whatever!) + # Can't think of a clean solution, so we hack away + block_suffix = "" + if not newdata[-1]: + block_suffix = "\n" + newdata = newdata[:-1] + return '\n'.join(comment_header+line for line in newdata) + block_suffix + def is_all_white(line): + """Return True if line is empty or all whitespace.""" + return re.match(r"^\s*$", line) is not None def get_indent(line): - return re.match(r"^(\s*)", line).group() + """Return the initial space or tab indent of line.""" + return re.match(r"^([ \t]*)", line).group() def get_comment_header(line): - m = re.match(r"^(\s*#*)", line) + """Return string with leading whitespace and '#' from line or ''. + + A null return indicates that the line is not a comment line. A non- + null return, such as ' #', will be used to find the other lines of + a comment block with the same indent. + """ + m = re.match(r"^([ \t]*#*)", line) if m is None: return "" return m.group(1) + +if __name__ == "__main__": + from test import support; support.use_resources = ['gui'] + import unittest + unittest.main('idlelib.idle_test.test_formatparagraph', + verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_formatparagraph.py b/Lib/idlelib/idle_test/test_formatparagraph.py new file mode 100644 index 00000000000..2ec8232335b --- /dev/null +++ b/Lib/idlelib/idle_test/test_formatparagraph.py @@ -0,0 +1,374 @@ +# Test the functions and main class method of FormatParagraph.py +import unittest +from idlelib import FormatParagraph as fp +from idlelib.EditorWindow import EditorWindow +from tkinter import Tk, Text, TclError +from test.support import requires + + +class Is_Get_Test(unittest.TestCase): + """Test the is_ and get_ functions""" + test_comment = '# This is a comment' + test_nocomment = 'This is not a comment' + trailingws_comment = '# This is a comment ' + leadingws_comment = ' # This is a comment' + leadingws_nocomment = ' This is not a comment' + + def test_is_all_white(self): + self.assertTrue(fp.is_all_white('')) + self.assertTrue(fp.is_all_white('\t\n\r\f\v')) + self.assertFalse(fp.is_all_white(self.test_comment)) + + def test_get_indent(self): + Equal = self.assertEqual + Equal(fp.get_indent(self.test_comment), '') + Equal(fp.get_indent(self.trailingws_comment), '') + Equal(fp.get_indent(self.leadingws_comment), ' ') + Equal(fp.get_indent(self.leadingws_nocomment), ' ') + + def test_get_comment_header(self): + Equal = self.assertEqual + # Test comment strings + Equal(fp.get_comment_header(self.test_comment), '#') + Equal(fp.get_comment_header(self.trailingws_comment), '#') + Equal(fp.get_comment_header(self.leadingws_comment), ' #') + # Test non-comment strings + Equal(fp.get_comment_header(self.leadingws_nocomment), ' ') + Equal(fp.get_comment_header(self.test_nocomment), '') + + +class FindTest(unittest.TestCase): + """Test the find_paragraph function in FormatParagraph. + + Using the runcase() function, find_paragraph() is called with 'mark' set at + multiple indexes before and inside the test paragraph. + + It appears that code with the same indentation as a quoted string is grouped + as part of the same paragraph, which is probably incorrect behavior. + """ + + @classmethod + def setUpClass(cls): + from idlelib.idle_test.mock_tk import Text + cls.text = Text() + + def runcase(self, inserttext, stopline, expected): + # Check that find_paragraph returns the expected paragraph when + # the mark index is set to beginning, middle, end of each line + # up to but not including the stop line + text = self.text + text.insert('1.0', inserttext) + for line in range(1, stopline): + linelength = int(text.index("%d.end" % line).split('.')[1]) + for col in (0, linelength//2, linelength): + tempindex = "%d.%d" % (line, col) + self.assertEqual(fp.find_paragraph(text, tempindex), expected) + text.delete('1.0', 'end') + + def test_find_comment(self): + comment = ( + "# Comment block with no blank lines before\n" + "# Comment line\n" + "\n") + self.runcase(comment, 3, ('1.0', '3.0', '#', comment[0:58])) + + comment = ( + "\n" + "# Comment block with whitespace line before and after\n" + "# Comment line\n" + "\n") + self.runcase(comment, 4, ('2.0', '4.0', '#', comment[1:70])) + + comment = ( + "\n" + " # Indented comment block with whitespace before and after\n" + " # Comment line\n" + "\n") + self.runcase(comment, 4, ('2.0', '4.0', ' #', comment[1:82])) + + comment = ( + "\n" + "# Single line comment\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:23])) + + comment = ( + "\n" + " # Single line comment with leading whitespace\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', ' #', comment[1:51])) + + comment = ( + "\n" + "# Comment immediately followed by code\n" + "x = 42\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:40])) + + comment = ( + "\n" + " # Indented comment immediately followed by code\n" + "x = 42\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', ' #', comment[1:53])) + + comment = ( + "\n" + "# Comment immediately followed by indented code\n" + " x = 42\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:49])) + + def test_find_paragraph(self): + teststring = ( + '"""String with no blank lines before\n' + 'String line\n' + '"""\n' + '\n') + self.runcase(teststring, 4, ('1.0', '4.0', '', teststring[0:53])) + + teststring = ( + "\n" + '"""String with whitespace line before and after\n' + 'String line.\n' + '"""\n' + '\n') + self.runcase(teststring, 5, ('2.0', '5.0', '', teststring[1:66])) + + teststring = ( + '\n' + ' """Indented string with whitespace before and after\n' + ' Comment string.\n' + ' """\n' + '\n') + self.runcase(teststring, 5, ('2.0', '5.0', ' ', teststring[1:85])) + + teststring = ( + '\n' + '"""Single line string."""\n' + '\n') + self.runcase(teststring, 3, ('2.0', '3.0', '', teststring[1:27])) + + teststring = ( + '\n' + ' """Single line string with leading whitespace."""\n' + '\n') + self.runcase(teststring, 3, ('2.0', '3.0', ' ', teststring[1:55])) + + +class ReformatFunctionTest(unittest.TestCase): + """Test the reformat_paragraph function without the editor window.""" + + def test_reformat_paragrah(self): + Equal = self.assertEqual + reform = fp.reformat_paragraph + hw = "O hello world" + Equal(reform(' ', 1), ' ') + Equal(reform("Hello world", 20), "Hello world") + + # Test without leading newline + Equal(reform(hw, 1), "O\nhello\nworld") + Equal(reform(hw, 6), "O\nhello\nworld") + Equal(reform(hw, 7), "O hello\nworld") + Equal(reform(hw, 12), "O hello\nworld") + Equal(reform(hw, 13), "O hello world") + + # Test with leading newline + hw = "\nO hello world" + Equal(reform(hw, 1), "\nO\nhello\nworld") + Equal(reform(hw, 6), "\nO\nhello\nworld") + Equal(reform(hw, 7), "\nO hello\nworld") + Equal(reform(hw, 12), "\nO hello\nworld") + Equal(reform(hw, 13), "\nO hello world") + + +class ReformatCommentTest(unittest.TestCase): + """Test the reformat_comment function without the editor window.""" + + def test_reformat_comment(self): + Equal = self.assertEqual + + # reformat_comment formats to a minimum of 20 characters + test_string = ( + " \"\"\"this is a test of a reformat for a triple quoted string" + " will it reformat to less than 70 characters for me?\"\"\"") + result = fp.reformat_comment(test_string, 70, " ") + expected = ( + " \"\"\"this is a test of a reformat for a triple quoted string will it\n" + " reformat to less than 70 characters for me?\"\"\"") + Equal(result, expected) + + test_comment = ( + "# this is a test of a reformat for a triple quoted string will " + "it reformat to less than 70 characters for me?") + result = fp.reformat_comment(test_comment, 70, "#") + expected = ( + "# this is a test of a reformat for a triple quoted string will it\n" + "# reformat to less than 70 characters for me?") + Equal(result, expected) + + +class FormatClassTest(unittest.TestCase): + def test_init_close(self): + instance = fp.FormatParagraph('editor') + self.assertEqual(instance.editwin, 'editor') + instance.close() + self.assertEqual(instance.editwin, None) + + +# For testing format_paragraph_event, Initialize FormatParagraph with +# a mock Editor with .text and .get_selection_indices. The text must +# be a Text wrapper that adds two methods + +# A real EditorWindow creates unneeded, time-consuming baggage and +# sometimes emits shutdown warnings like this: +# "warning: callback failed in WindowList +# : invalid command name ".55131368.windows". +# Calling EditorWindow._close in tearDownClass prevents this but causes +# other problems (windows left open). + +class TextWrapper: + def __init__(self, master): + self.text = Text(master=master) + def __getattr__(self, name): + return getattr(self.text, name) + def undo_block_start(self): pass + def undo_block_stop(self): pass + +class Editor: + def __init__(self, root): + self.text = TextWrapper(root) + get_selection_indices = EditorWindow. get_selection_indices + +class FormatEventTest(unittest.TestCase): + """Test the formatting of text inside a Text widget. + + This is done with FormatParagraph.format.paragraph_event, + which calls funtions in the module as appropriate. + """ + test_string = ( + " '''this is a test of a reformat for a triple " + "quoted string will it reformat to less than 70 " + "characters for me?'''\n") + multiline_test_string = ( + " '''The first line is under the max width.\n" + " The second line's length is way over the max width. It goes " + "on and on until it is over 100 characters long.\n" + " Same thing with the third line. It is also way over the max " + "width, but FormatParagraph will fix it.\n" + " '''\n") + multiline_test_comment = ( + "# The first line is under the max width.\n" + "# The second line's length is way over the max width. It goes on " + "and on until it is over 100 characters long.\n" + "# Same thing with the third line. It is also way over the max " + "width, but FormatParagraph will fix it.\n" + "# The fourth line is short like the first line.") + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + editor = Editor(root=cls.root) + cls.text = editor.text.text # Test code does not need the wrapper. + cls.formatter = fp.FormatParagraph(editor).format_paragraph_event + # Sets the insert mark just after the re-wrapped and inserted text. + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + + def test_short_line(self): + self.text.insert('1.0', "Short line\n") + self.formatter("Dummy") + self.assertEqual(self.text.get('1.0', 'insert'), "Short line\n" ) + self.text.delete('1.0', 'end') + + def test_long_line(self): + text = self.text + + # Set cursor ('insert' mark) to '1.0', within text. + text.insert('1.0', self.test_string) + text.mark_set('insert', '1.0') + self.formatter('ParameterDoesNothing') + result = text.get('1.0', 'insert') + # find function includes \n + expected = ( +" '''this is a test of a reformat for a triple quoted string will it\n" +" reformat to less than 70 characters for me?'''\n") # yes + self.assertEqual(result, expected) + text.delete('1.0', 'end') + + # Select from 1.11 to line end. + text.insert('1.0', self.test_string) + text.tag_add('sel', '1.11', '1.end') + self.formatter('ParameterDoesNothing') + result = text.get('1.0', 'insert') + # selection excludes \n + expected = ( +" '''this is a test of a reformat for a triple quoted string will it reformat\n" +" to less than 70 characters for me?'''") # no + self.assertEqual(result, expected) + text.delete('1.0', 'end') + + def test_multiple_lines(self): + text = self.text + # Select 2 long lines. + text.insert('1.0', self.multiline_test_string) + text.tag_add('sel', '2.0', '4.0') + self.formatter('ParameterDoesNothing') + result = text.get('2.0', 'insert') + expected = ( +" The second line's length is way over the max width. It goes on and\n" +" on until it is over 100 characters long. Same thing with the third\n" +" line. It is also way over the max width, but FormatParagraph will\n" +" fix it.\n") + self.assertEqual(result, expected) + text.delete('1.0', 'end') + + def test_comment_block(self): + text = self.text + + # Set cursor ('insert') to '1.0', within block. + text.insert('1.0', self.multiline_test_comment) + self.formatter('ParameterDoesNothing') + result = text.get('1.0', 'insert') + expected = ( +"# The first line is under the max width. The second line's length is\n" +"# way over the max width. It goes on and on until it is over 100\n" +"# characters long. Same thing with the third line. It is also way over\n" +"# the max width, but FormatParagraph will fix it. The fourth line is\n" +"# short like the first line.\n") + self.assertEqual(result, expected) + text.delete('1.0', 'end') + + # Select line 2, verify line 1 unaffected. + text.insert('1.0', self.multiline_test_comment) + text.tag_add('sel', '2.0', '3.0') + self.formatter('ParameterDoesNothing') + result = text.get('1.0', 'insert') + expected = ( +"# The first line is under the max width.\n" +"# The second line's length is way over the max width. It goes on and\n" +"# on until it is over 100 characters long.\n") + self.assertEqual(result, expected) + text.delete('1.0', 'end') + +# The following block worked with EditorWindow but fails with the mock. +# Lines 2 and 3 get pasted together even though the previous block left +# the previous line alone. More investigation is needed. +## # Select lines 3 and 4 +## text.insert('1.0', self.multiline_test_comment) +## text.tag_add('sel', '3.0', '5.0') +## self.formatter('ParameterDoesNothing') +## result = text.get('3.0', 'insert') +## expected = ( +##"# Same thing with the third line. It is also way over the max width,\n" +##"# but FormatParagraph will fix it. The fourth line is short like the\n" +##"# first line.\n") +## self.assertEqual(result, expected) +## text.delete('1.0', 'end') + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) diff --git a/Misc/NEWS b/Misc/NEWS index e5432b67980..920272789ec 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -113,6 +113,9 @@ Build IDLE ---- +- Issue #18226: Add docstrings and unittests for FormatParagraph.py. + Original patches by Todd Rovito and Phil Webster. + - Issue #18279: Format - Strip trailing whitespace no longer marks a file as changed when it has not been changed. This fix followed the addition of a test file originally written by Phil Webster (the issue's main goal). From 3e4d285ca89d1d1d8bb7f1ef8cce1699350fe375 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 10 Aug 2013 17:46:16 -0400 Subject: [PATCH 272/480] Issue #18429: Add user-oriented News entry about Format / Format Paragraph now working with comment block selections. Patch was part of 18226 patch. --- Misc/NEWS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 920272789ec..7e346d0f53a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -113,6 +113,10 @@ Build IDLE ---- +- Issue #18429: Format / Format Paragraph, now works when comment blocks + are selected. As with text blocks, this works best when the selection + only includes complete lines. + - Issue #18226: Add docstrings and unittests for FormatParagraph.py. Original patches by Todd Rovito and Phil Webster. From fe27bc40a70a1d54659dc39387aae881871454bb Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 10 Aug 2013 18:17:01 -0400 Subject: [PATCH 273/480] Issue #18676: Change 'positive' to 'non-negative' in queue.py put and get docstrings and ValueError messages. Patch by Zhongyue Luo --- Lib/Queue.py | 8 ++++---- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/Queue.py b/Lib/Queue.py index 2db8d769665..00364b39beb 100644 --- a/Lib/Queue.py +++ b/Lib/Queue.py @@ -109,7 +109,7 @@ def put(self, item, block=True, timeout=None): If optional args 'block' is true and 'timeout' is None (the default), block if necessary until a free slot is available. If 'timeout' is - a positive number, it blocks at most 'timeout' seconds and raises + a non-negative number, it blocks at most 'timeout' seconds and raises the Full exception if no free slot was available within that time. Otherwise ('block' is false), put an item on the queue if a free slot is immediately available, else raise the Full exception ('timeout' @@ -125,7 +125,7 @@ def put(self, item, block=True, timeout=None): while self._qsize() == self.maxsize: self.not_full.wait() elif timeout < 0: - raise ValueError("'timeout' must be a positive number") + raise ValueError("'timeout' must be a non-negative number") else: endtime = _time() + timeout while self._qsize() == self.maxsize: @@ -152,7 +152,7 @@ def get(self, block=True, timeout=None): If optional args 'block' is true and 'timeout' is None (the default), block if necessary until an item is available. If 'timeout' is - a positive number, it blocks at most 'timeout' seconds and raises + a non-negative number, it blocks at most 'timeout' seconds and raises the Empty exception if no item was available within that time. Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored @@ -167,7 +167,7 @@ def get(self, block=True, timeout=None): while not self._qsize(): self.not_empty.wait() elif timeout < 0: - raise ValueError("'timeout' must be a positive number") + raise ValueError("'timeout' must be a non-negative number") else: endtime = _time() + timeout while not self._qsize(): diff --git a/Misc/ACKS b/Misc/ACKS index 736887d62d8..fae831e85cd 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -629,6 +629,7 @@ Ray Loyzaga Lukas Lueg Loren Luke Fredrik Lundh +Zhongyue Luo Mark Lutz Jim Lynch Mikael Lyngvig diff --git a/Misc/NEWS b/Misc/NEWS index 7e346d0f53a..c9b7fdc6d25 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,9 @@ Core and Builtins Library ------- +- Issue #18676: Change 'positive' to 'non-negative' in queue.py put and get + docstrings and ValueError messages. Patch by Zhongyue Luo + - Issue #17998: Fix an internal error in regular expression engine. - Issue #17557: Fix os.getgroups() to work with the modified behavior of From 0dfa6a72f6f9553541f41a05dc294743f4b3869e Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 11 Aug 2013 13:04:50 +0300 Subject: [PATCH 274/480] #18663: document that assertAlmostEqual also works when the values are equal and add tests. --- Doc/library/unittest.rst | 2 +- Lib/unittest/test/test_assertions.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 1bd3819bc2d..f36a44deccf 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1017,7 +1017,7 @@ Test cases like the :func:`round` function) and not *significant digits*. If *delta* is supplied instead of *places* then the difference - between *first* and *second* must be less (or more) than *delta*. + between *first* and *second* must be less or equal to (or greater than) *delta*. Supplying both *delta* and *places* raises a ``TypeError``. diff --git a/Lib/unittest/test/test_assertions.py b/Lib/unittest/test/test_assertions.py index e1ba6147066..e8f0f64b4cf 100644 --- a/Lib/unittest/test/test_assertions.py +++ b/Lib/unittest/test/test_assertions.py @@ -33,6 +33,10 @@ def test_AmostEqualWithDelta(self): self.assertNotAlmostEqual(1.1, 1.0, delta=0.05) self.assertNotAlmostEqual(1.0, 1.1, delta=0.05) + self.assertAlmostEqual(1.0, 1.0, delta=0.5) + self.assertRaises(self.failureException, self.assertNotAlmostEqual, + 1.0, 1.0, delta=0.5) + self.assertRaises(self.failureException, self.assertAlmostEqual, 1.1, 1.0, delta=0.05) self.assertRaises(self.failureException, self.assertNotAlmostEqual, From f0682bb84d8889317e7db218fc5e4bdad1a580e3 Mon Sep 17 00:00:00 2001 From: Felix Crux Date: Mon, 12 Aug 2013 17:39:51 -0400 Subject: [PATCH 275/480] Documenting that json.load may raise a ValueError. Issue #18680: JSONDecoder should document that it raises a ValueError for malformed data --- Doc/library/json.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 6e1a9c3d2ed..caee9531dba 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -348,6 +348,8 @@ Encoders and Decoders those with character codes in the 0-31 range, including ``'\t'`` (tab), ``'\n'``, ``'\r'`` and ``'\0'``. + If the data being deserialized is not a valid JSON document, a + :exc:`ValueError` will be raised. .. method:: decode(s) From eef45cd9df98d6174e398ff12a261a34defd07cb Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Mon, 12 Aug 2013 22:24:43 -0700 Subject: [PATCH 276/480] Increasing test coverage of ftplib. Patch by Muhammad Jehanzeb --- Lib/test/test_ftplib.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index c82e8a69bf3..535bc49f061 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -474,6 +474,14 @@ def test_mkd(self): def test_rmd(self): self.client.rmd('foo') + def test_cwd(self): + dir = self.client.cwd('/foo') + self.assertEqual(dir, '250 cwd ok') + + def test_mkd(self): + dir = self.client.mkd('/foo') + self.assertEqual(dir, '/foo') + def test_pwd(self): dir = self.client.pwd() self.assertEqual(dir, 'pwd ok') From ca1cffd4318097435417db991837c1c8ce35a90e Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 13 Aug 2013 01:11:56 -0700 Subject: [PATCH 277/480] Issue 18724: Fix typo noticed by Susan Tan. --- Doc/library/smtplib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 4f3b572aab4..e145659f62e 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -79,7 +79,7 @@ A nice selection of exceptions is defined as well: .. exception:: SMTPException - The base exception class for all the other excpetions provided by this + The base exception class for all the other exceptions provided by this module. From 5cbb76e6516e4ff72aa700c230ac3adea9bede63 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Tue, 13 Aug 2013 12:53:40 +0300 Subject: [PATCH 278/480] =?UTF-8?q?#18687:=20remove=20obsolete=20comment.?= =?UTF-8?q?=20=20Patch=20by=20F=C3=A9vry=20Thibault.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/test/leakers/test_ctypes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/leakers/test_ctypes.py b/Lib/test/leakers/test_ctypes.py index 0f9a2cdc9a1..7d7e9ff3a11 100644 --- a/Lib/test/leakers/test_ctypes.py +++ b/Lib/test/leakers/test_ctypes.py @@ -1,6 +1,5 @@ # Taken from Lib/ctypes/test/test_keeprefs.py, PointerToStructure.test(). -# When this leak is fixed, remember to remove from Misc/build.sh LEAKY_TESTS. from ctypes import Structure, c_int, POINTER import gc From efb698e7a5c605fcb89b437aac4c80058958fc0e Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 13 Aug 2013 20:23:56 +0200 Subject: [PATCH 279/480] Add NEWS entry for 0f17aed78168 (issue #16248) --- Misc/NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 5e4a1bee3c6..d2d3b08a2ac 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,8 @@ Core and Builtins Library ------- +- Fix tkinter regression introduced by the security fix in issue #16248. + - Issue #18676: Change 'positive' to 'non-negative' in queue.py put and get docstrings and ValueError messages. Patch by Zhongyue Luo From 427f2100c282053115f841eff41cd46a03073aa5 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Tue, 13 Aug 2013 19:51:04 -0400 Subject: [PATCH 280/480] Issue #18425: Add docstrings to IdleHistory.py. Remove redundant 'history_' prefix from two attributes and two methods of History class. --- Lib/idlelib/IdleHistory.py | 48 +++++++++++++++++++++++++++----------- Lib/idlelib/PyShell.py | 2 +- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/Lib/idlelib/IdleHistory.py b/Lib/idlelib/IdleHistory.py index 983a1406d44..25fe6e4ad65 100644 --- a/Lib/idlelib/IdleHistory.py +++ b/Lib/idlelib/IdleHistory.py @@ -1,23 +1,41 @@ +"Implement Idle Shell history mechanism with History class" + from idlelib.configHandler import idleConf class History: + ''' Implement Idle Shell history mechanism. + store - Store source statement (called from PyShell.resetoutput). + fetch - Fetch stored statement matching prefix already entered. + history_next - Bound to <> event (default Alt-N). + history_prev - Bound to <> event (default Alt-P). + ''' def __init__(self, text, output_sep = "\n"): + '''Initialize data attributes and bind event methods. + + .text - Idle wrapper of tk Text widget, with .bell(). + .history - source statements, possibly with multiple lines. + .prefix - source already entered at prompt; filters history list. + .pointer - index into history. + .cyclic - wrap around history list (or not). + ''' self.text = text self.history = [] - self.history_prefix = None - self.history_pointer = None + self.prefix = None + self.pointer = None self.output_sep = output_sep self.cyclic = idleConf.GetOption("main", "History", "cyclic", 1, "bool") text.bind("<>", self.history_prev) text.bind("<>", self.history_next) def history_next(self, event): - self.history_do(0) + "Fetch later statement; start with ealiest if cyclic." + self.fetch(reverse=False) return "break" def history_prev(self, event): - self.history_do(1) + "Fetch earlier statement; start with most recent." + self.fetch(reverse=True) return "break" def _get_source(self, start, end): @@ -30,10 +48,11 @@ def _put_source(self, where, source): output = self.output_sep.join(source.split("\n")) self.text.insert(where, output) - def history_do(self, reverse): + def fetch(self, reverse): + "Fetch statememt and enter into text at cursor." nhist = len(self.history) - pointer = self.history_pointer - prefix = self.history_prefix + pointer = self.pointer + prefix = self.prefix if pointer is not None and prefix is not None: if self.text.compare("insert", "!=", "end-1c") or \ self._get_source("iomark", "end-1c") != self.history[pointer]: @@ -41,10 +60,10 @@ def history_do(self, reverse): if pointer is None or prefix is None: prefix = self._get_source("iomark", "end-1c") if reverse: - pointer = nhist + pointer = nhist # will be decremented else: if self.cyclic: - pointer = -1 + pointer = -1 # will be incremented else: self.text.bell() return @@ -72,10 +91,11 @@ def history_do(self, reverse): self.text.mark_set("insert", "end-1c") self.text.see("insert") self.text.tag_remove("sel", "1.0", "end") - self.history_pointer = pointer - self.history_prefix = prefix + self.pointer = pointer + self.prefix = prefix - def history_store(self, source): + def store(self, source): + "Store Shell input statement into history list." source = source.strip() if len(source) > 2: # avoid duplicates @@ -84,5 +104,5 @@ def history_store(self, source): except ValueError: pass self.history.append(source) - self.history_pointer = None - self.history_prefix = None + self.pointer = None + self.prefix = None diff --git a/Lib/idlelib/PyShell.py b/Lib/idlelib/PyShell.py index 0acfe4a95a4..7a9a0bc01b9 100644 --- a/Lib/idlelib/PyShell.py +++ b/Lib/idlelib/PyShell.py @@ -1278,7 +1278,7 @@ def showprompt(self): def resetoutput(self): source = self.text.get("iomark", "end-1c") if self.history: - self.history.history_store(source) + self.history.store(source) if self.text.get("end-2c") != "\n": self.text.insert("end-1c", "\n") self.text.mark_set("iomark", "end-1c") From 239e6046da50d757ef83d0e8d8a0ab46b92aaa2f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 13 Aug 2013 18:34:49 -0700 Subject: [PATCH 281/480] Issue 18719: Remove a false optimization Remove an unused early-out test from the critical path for dict and set lookups. When the strings already have matching lengths and hashes, there is no additional information gained by checking the first characters (the probability of a mismatch is already known to be less than 1 in 2**64). --- Objects/stringobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/stringobject.c b/Objects/stringobject.c index b80ef87b0d8..e1ea3cd80f7 100644 --- a/Objects/stringobject.c +++ b/Objects/stringobject.c @@ -1255,7 +1255,6 @@ _PyString_Eq(PyObject *o1, PyObject *o2) PyStringObject *a = (PyStringObject*) o1; PyStringObject *b = (PyStringObject*) o2; return Py_SIZE(a) == Py_SIZE(b) - && *a->ob_sval == *b->ob_sval && memcmp(a->ob_sval, b->ob_sval, Py_SIZE(a)) == 0; } From 4b6beeaab7e6eef84e645b403825a8325afc8fd9 Mon Sep 17 00:00:00 2001 From: David Wolever Date: Wed, 14 Aug 2013 14:33:54 -0400 Subject: [PATCH 282/480] Issue #18718: datetime documentation contradictory on leap second support --- Doc/library/datetime.rst | 10 +++------- Misc/NEWS | 3 +++ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index be41e1210db..4e922f03a08 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1664,7 +1664,7 @@ platforms. Regardless of platform, years before 1900 cannot be used. | ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 61 | \(3) | +| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(3) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(4) | @@ -1728,12 +1728,8 @@ Notes: the output hour field if the ``%I`` directive is used to parse the hour. (3) - The range really is ``0`` to ``61``; according to the Posix standard this - accounts for leap seconds and the (very rare) double leap seconds. - The :mod:`time` module may produce and does accept leap seconds since - it is based on the Posix standard, but the :mod:`datetime` module - does not accept leap seconds in :meth:`strptime` input nor will it - produce them in :func:`strftime` output. + Unlike the :mod:`time` module, the :mod:`datetime` module does not support + leap seconds. (4) ``%f`` is an extension to the set of format characters in the C standard diff --git a/Misc/NEWS b/Misc/NEWS index d2d3b08a2ac..2aad9bbd066 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -166,7 +166,10 @@ Tests Documentation ------------- +- Issue #18718: datetime documentation contradictory on leap second support. + - Issue #17701: Improving strftime documentation. + - Issue #17844: Refactor a documentation of Python specific encodings. Add links to encoders and decoders for binary-to-binary codecs. From 2effdb99d3a59639ab85cad4e651118d8af979af Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Thu, 15 Aug 2013 14:31:55 -0400 Subject: [PATCH 283/480] Issue #18425: Unittests for idlelib.IdleHistory. First patch by R. Jayakrishnan. Issue #18425: Unittests for idlelib.IdleHistory. First patch by R. Jayakrishnan. Issue #18425: Unittests for idlelib.IdleHistory. First patch by R. Jayakrishnan. Issue #18425: Unittests for idlelib.IdleHistory. First patch by R. Jayakrishnan. Issue #18425: Unittests for idlelib.IdleHistory. First patch by R. Jayakrishnan. Issue #18425: Unittests for idlelib.IdleHistory. First patch by R. Jayakrishnan. Issue #18425: Unittests for idlelib.IdleHistory. First patch by R. Jayakrishnan. Issue #18425: Unittests for idlelib.IdleHistory. First patch by R. Jayakrishnan. Issue #18425: Unittests for idlelib.IdleHistory. First patch by R. Jayakrishnan. --- Lib/idlelib/IdleHistory.py | 18 ++- Lib/idlelib/idle_test/test_idlehistory.py | 164 ++++++++++++++++++++++ 2 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 Lib/idlelib/idle_test/test_idlehistory.py diff --git a/Lib/idlelib/IdleHistory.py b/Lib/idlelib/IdleHistory.py index 25fe6e4ad65..dc1ac8b5d2e 100644 --- a/Lib/idlelib/IdleHistory.py +++ b/Lib/idlelib/IdleHistory.py @@ -49,7 +49,13 @@ def _put_source(self, where, source): self.text.insert(where, output) def fetch(self, reverse): - "Fetch statememt and enter into text at cursor." + '''Fetch statememt and replace current line in text widget. + + Set prefix and pointer as needed for successive fetches. + Reset them to None, None when returning to the start line. + Sound bell when return to start line or cannot leave a line + because cyclic is False. + ''' nhist = len(self.history) pointer = self.pointer prefix = self.prefix @@ -64,7 +70,7 @@ def fetch(self, reverse): else: if self.cyclic: pointer = -1 # will be incremented - else: + else: # abort history_next self.text.bell() return nprefix = len(prefix) @@ -75,7 +81,7 @@ def fetch(self, reverse): pointer = pointer + 1 if pointer < 0 or pointer >= nhist: self.text.bell() - if not self.cyclic and pointer < 0: + if not self.cyclic and pointer < 0: # abort history_prev return else: if self._get_source("iomark", "end-1c") != prefix: @@ -106,3 +112,9 @@ def store(self, source): self.history.append(source) self.pointer = None self.prefix = None + +if __name__ == "__main__": + from test import support + support.use_resources = ['gui'] + from unittest import main + main('idlelib.idle_test.test_idlehistory', verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_idlehistory.py b/Lib/idlelib/idle_test/test_idlehistory.py new file mode 100644 index 00000000000..56256b8ebd3 --- /dev/null +++ b/Lib/idlelib/idle_test/test_idlehistory.py @@ -0,0 +1,164 @@ +import unittest +from test.test_support import requires + +import Tkinter as tk +from Tkinter import Text as tkText +from idlelib.idle_test.mock_tk import Text as mkText +from idlelib.IdleHistory import History +from idlelib.configHandler import idleConf + +line1 = 'a = 7' +line2 = 'b = a' + +class StoreTest(unittest.TestCase): + '''Tests History.__init__ and History.store with mock Text''' + + @classmethod + def setUpClass(cls): + cls.text = mkText() + cls.history = History(cls.text) + + def tearDown(self): + self.text.delete('1.0', 'end') + self.history.history = [] + + def test_init(self): + self.assertIs(self.history.text, self.text) + self.assertEqual(self.history.history, []) + self.assertIsNone(self.history.prefix) + self.assertIsNone(self.history.pointer) + self.assertEqual(self.history.cyclic, + idleConf.GetOption("main", "History", "cyclic", 1, "bool")) + + def test_store_short(self): + self.history.store('a') + self.assertEqual(self.history.history, []) + self.history.store(' a ') + self.assertEqual(self.history.history, []) + + def test_store_dup(self): + self.history.store(line1) + self.assertEqual(self.history.history, [line1]) + self.history.store(line2) + self.assertEqual(self.history.history, [line1, line2]) + self.history.store(line1) + self.assertEqual(self.history.history, [line2, line1]) + + def test_store_reset(self): + self.history.prefix = line1 + self.history.pointer = 0 + self.history.store(line2) + self.assertIsNone(self.history.prefix) + self.assertIsNone(self.history.pointer) + + +class TextWrapper: + def __init__(self, master): + self.text = tkText(master=master) + self._bell = False + def __getattr__(self, name): + return getattr(self.text, name) + def bell(self): + self._bell = True + +class FetchTest(unittest.TestCase): + '''Test History.fetch with wrapped tk.Text. + ''' + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = tk.Tk() + + def setUp(self): + self.text = text = TextWrapper(self.root) + text.insert('1.0', ">>> ") + text.mark_set('iomark', '1.4') + text.mark_gravity('iomark', 'left') + self.history = History(text) + self.history.history = [line1, line2] + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + + def fetch_test(self, reverse, line, prefix, index, bell=False): + # Perform one fetch as invoked by Alt-N or Alt-P + # Test the result. The line test is the most important. + # The last two are diagnostic of fetch internals. + History = self.history + History.fetch(reverse) + Equal = self.assertEqual + Equal(self.text.get('iomark', 'end-1c'), line) + Equal(self.text._bell, bell) + if bell: + self.text._bell = False + Equal(History.prefix, prefix) + Equal(History.pointer, index) + + def test_fetch_prev_cyclic(self): + prefix = '' + test = self.fetch_test + test(True, line2, prefix, 1) + test(True, line1, prefix, 0) + test(True, prefix, None, None, bell=True) + + def test_fetch_next_cyclic(self): + prefix = '' + test = self.fetch_test + test(False, line1, prefix, 0) + test(False, line2, prefix, 1) + test(False, prefix, None, None, bell=True) + + # Prefix 'a' tests skip line2, which starts with 'b' + def test_fetch_prev_prefix(self): + prefix = 'a' + self.text.insert('iomark', prefix) + self.fetch_test(True, line1, prefix, 0) + self.fetch_test(True, prefix, None, None, bell=True) + + def test_fetch_next_prefix(self): + prefix = 'a' + self.text.insert('iomark', prefix) + self.fetch_test(False, line1, prefix, 0) + self.fetch_test(False, prefix, None, None, bell=True) + + def test_fetch_prev_noncyclic(self): + prefix = '' + self.history.cyclic = False + test = self.fetch_test + test(True, line2, prefix, 1) + test(True, line1, prefix, 0) + test(True, line1, prefix, 0, bell=True) + + def test_fetch_next_noncyclic(self): + prefix = '' + self.history.cyclic = False + test = self.fetch_test + test(False, prefix, None, None, bell=True) + test(True, line2, prefix, 1) + test(False, prefix, None, None, bell=True) + test(False, prefix, None, None, bell=True) + + def test_fetch_cursor_move(self): + # Move cursor after fetch + self.history.fetch(reverse=True) # initialization + self.text.mark_set('insert', 'iomark') + self.fetch_test(True, line2, None, None, bell=True) + + def test_fetch_edit(self): + # Edit after fetch + self.history.fetch(reverse=True) # initialization + self.text.delete('iomark', 'insert', ) + self.text.insert('iomark', 'a =') + self.fetch_test(True, line1, 'a =', 0) # prefix is reset + + def test_history_prev_next(self): + # Minimally test functions bound to events + self.history.history_prev('dummy event') + self.assertEqual(self.history.pointer, 1) + self.history.history_next('dummy event') + self.assertEqual(self.history.pointer, None) + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) From e29bb722c7215801737755657ed585ac61c86b15 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Thu, 15 Aug 2013 15:07:58 -0400 Subject: [PATCH 284/480] Issue #18226: Fix ImportError and subsequent TypeError in 2.7 backport. Running py27\PCbuild> python_d -m test.regrtest -ugui test_idle ignores the exceptions and gives no indication of a problem (fixed in 3.3). --- Lib/idlelib/idle_test/test_formatparagraph.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/idlelib/idle_test/test_formatparagraph.py b/Lib/idlelib/idle_test/test_formatparagraph.py index 2ec8232335b..fca9540e92c 100644 --- a/Lib/idlelib/idle_test/test_formatparagraph.py +++ b/Lib/idlelib/idle_test/test_formatparagraph.py @@ -2,8 +2,8 @@ import unittest from idlelib import FormatParagraph as fp from idlelib.EditorWindow import EditorWindow -from tkinter import Tk, Text, TclError -from test.support import requires +from Tkinter import Tk, Text, TclError +from test.test_support import requires class Is_Get_Test(unittest.TestCase): @@ -238,7 +238,7 @@ def undo_block_stop(self): pass class Editor: def __init__(self, root): self.text = TextWrapper(root) - get_selection_indices = EditorWindow. get_selection_indices + get_selection_indices = EditorWindow. get_selection_indices.im_func class FormatEventTest(unittest.TestCase): """Test the formatting of text inside a Text widget. From d418c368290b7fbc25da8f5d47c298740c992469 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Thu, 15 Aug 2013 16:19:07 -0400 Subject: [PATCH 285/480] Issue #18732: Remove unused* parameter output_sep from IdleHistory.History and paired splits and joins that do nothing when output_sep is its default \n. *It in unused in that the class in only instantiated once, with the default. Make a few other changes in .fetch and its test. --- Lib/idlelib/IdleHistory.py | 32 +++++++---------------- Lib/idlelib/idle_test/test_idlehistory.py | 2 ++ 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/Lib/idlelib/IdleHistory.py b/Lib/idlelib/IdleHistory.py index dc1ac8b5d2e..931d5b28e43 100644 --- a/Lib/idlelib/IdleHistory.py +++ b/Lib/idlelib/IdleHistory.py @@ -10,7 +10,7 @@ class History: history_next - Bound to <> event (default Alt-N). history_prev - Bound to <> event (default Alt-P). ''' - def __init__(self, text, output_sep = "\n"): + def __init__(self, text): '''Initialize data attributes and bind event methods. .text - Idle wrapper of tk Text widget, with .bell(). @@ -23,7 +23,6 @@ def __init__(self, text, output_sep = "\n"): self.history = [] self.prefix = None self.pointer = None - self.output_sep = output_sep self.cyclic = idleConf.GetOption("main", "History", "cyclic", 1, "bool") text.bind("<>", self.history_prev) text.bind("<>", self.history_next) @@ -38,16 +37,6 @@ def history_prev(self, event): self.fetch(reverse=True) return "break" - def _get_source(self, start, end): - # Get source code from start index to end index. Lines in the - # text control may be separated by sys.ps2 . - lines = self.text.get(start, end).split(self.output_sep) - return "\n".join(lines) - - def _put_source(self, where, source): - output = self.output_sep.join(source.split("\n")) - self.text.insert(where, output) - def fetch(self, reverse): '''Fetch statememt and replace current line in text widget. @@ -61,10 +50,11 @@ def fetch(self, reverse): prefix = self.prefix if pointer is not None and prefix is not None: if self.text.compare("insert", "!=", "end-1c") or \ - self._get_source("iomark", "end-1c") != self.history[pointer]: + self.text.get("iomark", "end-1c") != self.history[pointer]: pointer = prefix = None + self.text.mark_set("insert", "end-1c") # != after cursor move if pointer is None or prefix is None: - prefix = self._get_source("iomark", "end-1c") + prefix = self.text.get("iomark", "end-1c") if reverse: pointer = nhist # will be decremented else: @@ -75,26 +65,22 @@ def fetch(self, reverse): return nprefix = len(prefix) while 1: - if reverse: - pointer = pointer - 1 - else: - pointer = pointer + 1 + pointer += -1 if reverse else 1 if pointer < 0 or pointer >= nhist: self.text.bell() if not self.cyclic and pointer < 0: # abort history_prev return else: - if self._get_source("iomark", "end-1c") != prefix: + if self.text.get("iomark", "end-1c") != prefix: self.text.delete("iomark", "end-1c") - self._put_source("iomark", prefix) + self.text.insert("iomark", prefix) pointer = prefix = None break item = self.history[pointer] if item[:nprefix] == prefix and len(item) > nprefix: self.text.delete("iomark", "end-1c") - self._put_source("iomark", item) + self.text.insert("iomark", item) break - self.text.mark_set("insert", "end-1c") self.text.see("insert") self.text.tag_remove("sel", "1.0", "end") self.pointer = pointer @@ -114,7 +100,7 @@ def store(self, source): self.prefix = None if __name__ == "__main__": - from test import support + from test import test_support as support support.use_resources = ['gui'] from unittest import main main('idlelib.idle_test.test_idlehistory', verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_idlehistory.py b/Lib/idlelib/idle_test/test_idlehistory.py index 56256b8ebd3..c15a9266d77 100644 --- a/Lib/idlelib/idle_test/test_idlehistory.py +++ b/Lib/idlelib/idle_test/test_idlehistory.py @@ -87,6 +87,7 @@ def fetch_test(self, reverse, line, prefix, index, bell=False): # The last two are diagnostic of fetch internals. History = self.history History.fetch(reverse) + Equal = self.assertEqual Equal(self.text.get('iomark', 'end-1c'), line) Equal(self.text._bell, bell) @@ -94,6 +95,7 @@ def fetch_test(self, reverse, line, prefix, index, bell=False): self.text._bell = False Equal(History.prefix, prefix) Equal(History.pointer, index) + Equal(self.text.compare("insert", '==', "end-1c"), 1) def test_fetch_prev_cyclic(self): prefix = '' From d0a9696bc65b67c186dafc9367fa36e11ea2e678 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 16 Aug 2013 19:19:40 +0200 Subject: [PATCH 286/480] Issue #16190: fix random module recommendation to use ssl.RAND_bytes(). --- Doc/library/random.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/library/random.rst b/Doc/library/random.rst index b1df231e736..1bc998940dc 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -60,6 +60,13 @@ The :mod:`random` module also provides the :class:`SystemRandom` class which uses the system function :func:`os.urandom` to generate random numbers from sources provided by the operating system. +.. warning:: + + The pseudo-random generators of this module should not be used for + security purposes. Use :func:`os.urandom` or :class:`SystemRandom` if + you require a cryptographically secure pseudo-random number generator. + + Bookkeeping functions: From 6a57acd2dfde5568f566ba34f95319dcf75ebec1 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Fri, 16 Aug 2013 21:32:25 +0300 Subject: [PATCH 287/480] #18707: point to Doc/README.txt in the README file. Patch by Madison May. --- README | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README b/README index 014028dacbf..2b9fc69252f 100644 --- a/README +++ b/README @@ -89,6 +89,13 @@ reStructuredText (2.6+) formats; the LaTeX and reStructuredText versions are primarily for documentation authors, translators, and people with special formatting requirements. +If you would like to contribute to the development of Python, relevant +documentation is available at: + + http://docs.python.org/devguide/ + +For information about building Python's documentation, refer to Doc/README.txt. + Web sites --------- From 9e801ba3e4d0da456704f63967b377dfeee297dc Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 16 Aug 2013 20:44:38 +0200 Subject: [PATCH 288/480] Issue #18756: Improve error reporting in os.urandom() when the failure is due to something else than /dev/urandom not existing. --- Lib/test/test_os.py | 18 ++++++++++++++++++ Misc/NEWS | 4 ++++ Python/random.c | 8 ++++++-- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index bdbeb0107bb..ccc58d4fbb7 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -10,6 +10,10 @@ import signal import subprocess import time +try: + import resource +except ImportError: + resource = None from test import test_support import mmap @@ -563,6 +567,20 @@ def test_urandom_subprocess(self): data2 = self.get_urandom_subprocess(16) self.assertNotEqual(data1, data2) + @unittest.skipUnless(resource, "test requires the resource module") + def test_urandom_failure(self): + soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + resource.setrlimit(resource.RLIMIT_NOFILE, (1, hard_limit)) + try: + with self.assertRaises(OSError) as cm: + os.urandom(16) + self.assertEqual(cm.exception.errno, errno.EMFILE) + finally: + # We restore the old limit as soon as possible. If doing it + # using addCleanup(), code running in between would fail + # creating any file descriptor. + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) + def test_execvpe_with_bad_arglist(self): self.assertRaises(ValueError, os.execvpe, 'notepad', [], None) diff --git a/Misc/NEWS b/Misc/NEWS index 2aad9bbd066..e5101f5fb0c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,10 @@ Core and Builtins Library ------- +- Issue #18756: Improve error reporting in os.urandom() when the failure + is due to something else than /dev/urandom not existing (for example, + exhausting the file descriptor limit). + - Fix tkinter regression introduced by the security fix in issue #16248. - Issue #18676: Change 'positive' to 'non-negative' in queue.py put and get diff --git a/Python/random.c b/Python/random.c index 825260f36b2..73e3cc3551f 100644 --- a/Python/random.c +++ b/Python/random.c @@ -165,8 +165,12 @@ dev_urandom_python(char *buffer, Py_ssize_t size) Py_END_ALLOW_THREADS if (fd < 0) { - PyErr_SetString(PyExc_NotImplementedError, - "/dev/urandom (or equivalent) not found"); + if (errno == ENOENT || errno == ENXIO || + errno == ENODEV || errno == EACCES) + PyErr_SetString(PyExc_NotImplementedError, + "/dev/urandom (or equivalent) not found"); + else + PyErr_SetFromErrno(PyExc_OSError); return -1; } From 85266e0a536d2f00b1fbd21480c771403323fa12 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 16 Aug 2013 20:56:12 +0200 Subject: [PATCH 289/480] test_execvpe_with_bad_arglist shouldn't be in URandomTests --- Lib/test/test_os.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index ccc58d4fbb7..bc7dfbea056 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -581,9 +581,13 @@ def test_urandom_failure(self): # creating any file descriptor. resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) + +class ExecvpeTests(unittest.TestCase): + def test_execvpe_with_bad_arglist(self): self.assertRaises(ValueError, os.execvpe, 'notepad', [], None) + class Win32ErrorTests(unittest.TestCase): def test_rename(self): self.assertRaises(WindowsError, os.rename, test_support.TESTFN, test_support.TESTFN+".bak") @@ -870,6 +874,7 @@ def test_main(): MakedirTests, DevNullTests, URandomTests, + ExecvpeTests, Win32ErrorTests, TestInvalidFD, PosixUidGidTests, From d21eb7a9a47e2f4cb68cf5f52f6b50b5400f6c7e Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 17 Aug 2013 00:54:47 +0200 Subject: [PATCH 290/480] Issue #18709: Fix CVE-2013-4238. The SSL module now handles NULL bytes inside subjectAltName correctly. Formerly the module has used OpenSSL's GENERAL_NAME_print() function to get the string represention of ASN.1 strings for rfc822Name (email), dNSName (DNS) and uniformResourceIdentifier (URI). --- Lib/test/nullbytecert.pem | 90 +++++++++++++++++++++++++++++++++++++++ Lib/test/test_ssl.py | 29 ++++++++++++- Misc/NEWS | 6 +++ Modules/_ssl.c | 66 +++++++++++++++++++++++++--- 4 files changed, 183 insertions(+), 8 deletions(-) create mode 100644 Lib/test/nullbytecert.pem diff --git a/Lib/test/nullbytecert.pem b/Lib/test/nullbytecert.pem new file mode 100644 index 00000000000..447186c9508 --- /dev/null +++ b/Lib/test/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 75521990228..563f9293fef 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -25,6 +25,7 @@ HOST = test_support.HOST CERTFILE = None SVN_PYTHON_ORG_ROOT_CERT = None +NULLBYTECERT = None def handle_error(prefix): exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) @@ -123,6 +124,27 @@ def test_parse_cert(self): ('DNS', 'projects.forum.nokia.com')) ) + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if test_support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + self.assertEqual(p['subjectAltName'], + (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + ) + def test_DER_to_PEM(self): with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f: pem = f.read() @@ -1360,7 +1382,7 @@ def test_default_ciphers(self): def test_main(verbose=False): - global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, NOKIACERT + global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, NOKIACERT, NULLBYTECERT CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "keycert.pem") SVN_PYTHON_ORG_ROOT_CERT = os.path.join( @@ -1368,10 +1390,13 @@ def test_main(verbose=False): "https_svn_python_org_root.pem") NOKIACERT = os.path.join(os.path.dirname(__file__) or os.curdir, "nokia.pem") + NULLBYTECERT = os.path.join(os.path.dirname(__file__) or os.curdir, + "nullbytecert.pem") if (not os.path.exists(CERTFILE) or not os.path.exists(SVN_PYTHON_ORG_ROOT_CERT) or - not os.path.exists(NOKIACERT)): + not os.path.exists(NOKIACERT) or + not os.path.exists(NULLBYTECERT)): raise test_support.TestFailed("Can't read certificate files!") tests = [BasicTests, BasicSocketTests] diff --git a/Misc/NEWS b/Misc/NEWS index e5101f5fb0c..37ada36c0c6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,12 @@ Core and Builtins Library ------- +- Issue #18709: Fix CVE-2013-4238. The SSL module now handles NULL bytes + inside subjectAltName correctly. Formerly the module has used OpenSSL's + GENERAL_NAME_print() function to get the string represention of ASN.1 + strings for ``rfc822Name`` (email), ``dNSName`` (DNS) and + ``uniformResourceIdentifier`` (URI). + - Issue #18756: Improve error reporting in os.urandom() when the failure is due to something else than /dev/urandom not existing (for example, exhausting the file descriptor limit). diff --git a/Modules/_ssl.c b/Modules/_ssl.c index afcc0174ab8..b6c2f6b3223 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -738,13 +738,16 @@ _get_peer_alt_names (X509 *certificate) { ext->value->length)); for(j = 0; j < sk_GENERAL_NAME_num(names); j++) { - /* get a rendering of each name in the set of names */ + int gntype; + ASN1_STRING *as = NULL; name = sk_GENERAL_NAME_value(names, j); - if (name->type == GEN_DIRNAME) { - - /* we special-case DirName as a tuple of tuples of attributes */ + gntype = name-> type; + switch (gntype) { + case GEN_DIRNAME: + /* we special-case DirName as a tuple of + tuples of attributes */ t = PyTuple_New(2); if (t == NULL) { @@ -764,11 +767,61 @@ _get_peer_alt_names (X509 *certificate) { goto fail; } PyTuple_SET_ITEM(t, 1, v); + break; - } else { + case GEN_EMAIL: + case GEN_DNS: + case GEN_URI: + /* GENERAL_NAME_print() doesn't handle NULL bytes in ASN1_string + correctly, CVE-2013-4238 */ + t = PyTuple_New(2); + if (t == NULL) + goto fail; + switch (gntype) { + case GEN_EMAIL: + v = PyString_FromString("email"); + as = name->d.rfc822Name; + break; + case GEN_DNS: + v = PyString_FromString("DNS"); + as = name->d.dNSName; + break; + case GEN_URI: + v = PyString_FromString("URI"); + as = name->d.uniformResourceIdentifier; + break; + } + if (v == NULL) { + Py_DECREF(t); + goto fail; + } + PyTuple_SET_ITEM(t, 0, v); + v = PyString_FromStringAndSize((char *)ASN1_STRING_data(as), + ASN1_STRING_length(as)); + if (v == NULL) { + Py_DECREF(t); + goto fail; + } + PyTuple_SET_ITEM(t, 1, v); + break; + default: /* for everything else, we use the OpenSSL print form */ - + switch (gntype) { + /* check for new general name type */ + case GEN_OTHERNAME: + case GEN_X400: + case GEN_EDIPARTY: + case GEN_IPADD: + case GEN_RID: + break; + default: + if (PyErr_Warn(PyExc_RuntimeWarning, + "Unknown general name type") == -1) { + goto fail; + } + break; + } (void) BIO_reset(biobuf); GENERAL_NAME_print(biobuf, name); len = BIO_gets(biobuf, buf, sizeof(buf)-1); @@ -794,6 +847,7 @@ _get_peer_alt_names (X509 *certificate) { goto fail; } PyTuple_SET_ITEM(t, 1, v); + break; } /* and add that rendering to the list */ From 04a4f64327a87870340143d12823e33ba6c22025 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Sat, 17 Aug 2013 00:38:48 +0100 Subject: [PATCH 291/480] Issue #18759: Improved cross-references in logging documentation. --- Doc/howto/logging-cookbook.rst | 55 +++++++++++++-------------- Doc/howto/logging.rst | 64 +++++++++++++++++--------------- Doc/library/logging.config.rst | 14 ++++--- Doc/library/logging.handlers.rst | 14 ++++--- 4 files changed, 78 insertions(+), 69 deletions(-) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 00d0e809143..e5d75a64b9b 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -97,11 +97,11 @@ The output looks like this:: Multiple handlers and formatters -------------------------------- -Loggers are plain Python objects. The :func:`addHandler` method has no minimum -or maximum quota for the number of handlers you may add. Sometimes it will be -beneficial for an application to log all messages of all severities to a text -file while simultaneously logging errors or above to the console. To set this -up, simply configure the appropriate handlers. The logging calls in the +Loggers are plain Python objects. The :meth:`~Logger.addHandler` method has no +minimum or maximum quota for the number of handlers you may add. Sometimes it +will be beneficial for an application to log all messages of all severities to a +text file while simultaneously logging errors or above to the console. To set +this up, simply configure the appropriate handlers. The logging calls in the application code will remain unchanged. Here is a slight modification to the previous simple module-based configuration example:: @@ -395,8 +395,9 @@ printed on the console; on the server side, you should see something like:: Note that there are some security issues with pickle in some scenarios. If these affect you, you can use an alternative serialization scheme by overriding -the :meth:`makePickle` method and implementing your alternative there, as -well as adapting the above script to use your alternative serialization. +the :meth:`~handlers.SocketHandler.makePickle` method and implementing your +alternative there, as well as adapting the above script to use your alternative +serialization. .. _context-info: @@ -404,6 +405,8 @@ well as adapting the above script to use your alternative serialization. Adding contextual information to your logging output ---------------------------------------------------- +.. currentmodule:: logging + Sometimes you want logging output to contain contextual information in addition to the parameters passed to the logging call. For example, in a networked application, it may be desirable to log client-specific information @@ -445,9 +448,9 @@ information in the delegated call. Here's a snippet from the code of msg, kwargs = self.process(msg, kwargs) self.logger.debug(msg, *args, **kwargs) -The :meth:`process` method of :class:`LoggerAdapter` is where the contextual -information is added to the logging output. It's passed the message and -keyword arguments of the logging call, and it passes back (potentially) +The :meth:`~LoggerAdapter.process` method of :class:`LoggerAdapter` is where the +contextual information is added to the logging output. It's passed the message +and keyword arguments of the logging call, and it passes back (potentially) modified versions of these to use in the call to the underlying logger. The default implementation of this method leaves the message alone, but inserts an 'extra' key in the keyword argument whose value is the dict-like object @@ -459,8 +462,8 @@ merged into the :class:`LogRecord` instance's __dict__, allowing you to use customized strings with your :class:`Formatter` instances which know about the keys of the dict-like object. If you need a different method, e.g. if you want to prepend or append the contextual information to the message string, -you just need to subclass :class:`LoggerAdapter` and override :meth:`process` -to do what you need. Here is a simple example:: +you just need to subclass :class:`LoggerAdapter` and override +:meth:`~LoggerAdapter.process` to do what you need. Here is a simple example:: class CustomAdapter(logging.LoggerAdapter): """ @@ -569,25 +572,23 @@ threads in a single process *is* supported, logging to a single file from *multiple processes* is *not* supported, because there is no standard way to serialize access to a single file across multiple processes in Python. If you need to log to a single file from multiple processes, one way of doing this is -to have all the processes log to a :class:`SocketHandler`, and have a separate -process which implements a socket server which reads from the socket and logs -to file. (If you prefer, you can dedicate one thread in one of the existing -processes to perform this function.) :ref:`This section ` -documents this approach in more detail and includes a working socket receiver -which can be used as a starting point for you to adapt in your own -applications. +to have all the processes log to a :class:`~handlers.SocketHandler`, and have a +separate process which implements a socket server which reads from the socket +and logs to file. (If you prefer, you can dedicate one thread in one of the +existing processes to perform this function.) +:ref:`This section ` documents this approach in more detail and +includes a working socket receiver which can be used as a starting point for you +to adapt in your own applications. If you are using a recent version of Python which includes the :mod:`multiprocessing` module, you could write your own handler which uses the -:class:`Lock` class from this module to serialize access to the file from -your processes. The existing :class:`FileHandler` and subclasses do not make -use of :mod:`multiprocessing` at present, though they may do so in the future. -Note that at present, the :mod:`multiprocessing` module does not provide +:class:`~multiprocessing.Lock` class from this module to serialize access to the +file from your processes. The existing :class:`FileHandler` and subclasses do +not make use of :mod:`multiprocessing` at present, though they may do so in the +future. Note that at present, the :mod:`multiprocessing` module does not provide working lock functionality on all platforms (see http://bugs.python.org/issue3770). -.. currentmodule:: logging.handlers - Using file rotation ------------------- @@ -599,7 +600,7 @@ Sometimes you want to let a log file grow to a certain size, then open a new file and log to that. You may want to keep a certain number of these files, and when that many files have been created, rotate the files so that the number of files and the size of the files both remain bounded. For this usage pattern, the -logging package provides a :class:`RotatingFileHandler`:: +logging package provides a :class:`~handlers.RotatingFileHandler`:: import glob import logging @@ -650,7 +651,7 @@ An example dictionary-based configuration Below is an example of a logging configuration dictionary - it's taken from the `documentation on the Django project `_. -This dictionary is passed to :func:`~logging.config.dictConfig` to put the configuration into effect:: +This dictionary is passed to :func:`~config.dictConfig` to put the configuration into effect:: LOGGING = { 'version': 1, diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index f6e7153a5cb..1916c47bb7b 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -469,12 +469,13 @@ Handlers :class:`~logging.Handler` objects are responsible for dispatching the appropriate log messages (based on the log messages' severity) to the handler's -specified destination. Logger objects can add zero or more handler objects to -themselves with an :func:`addHandler` method. As an example scenario, an -application may want to send all log messages to a log file, all log messages -of error or higher to stdout, and all messages of critical to an email address. -This scenario requires three individual handlers where each handler is -responsible for sending messages of a specific severity to a specific location. +specified destination. :class:`Logger` objects can add zero or more handler +objects to themselves with an :meth:`~Logger.addHandler` method. As an example +scenario, an application may want to send all log messages to a log file, all +log messages of error or higher to stdout, and all messages of critical to an +email address. This scenario requires three individual handlers where each +handler is responsible for sending messages of a specific severity to a specific +location. The standard library includes quite a few handler types (see :ref:`useful-handlers`); the tutorials use mainly :class:`StreamHandler` and @@ -485,16 +486,17 @@ themselves with. The only handler methods that seem relevant for application developers who are using the built-in handler objects (that is, not creating custom handlers) are the following configuration methods: -* The :meth:`Handler.setLevel` method, just as in logger objects, specifies the +* The :meth:`~Handler.setLevel` method, just as in logger objects, specifies the lowest severity that will be dispatched to the appropriate destination. Why are there two :func:`setLevel` methods? The level set in the logger determines which severity of messages it will pass to its handlers. The level set in each handler determines which messages that handler will send on. -* :func:`setFormatter` selects a Formatter object for this handler to use. +* :meth:`~Handler.setFormatter` selects a Formatter object for this handler to + use. -* :func:`addFilter` and :func:`removeFilter` respectively configure and - deconfigure filter objects on handlers. +* :meth:`~Handler.addFilter` and :meth:`~Handler.removeFilter` respectively + configure and deconfigure filter objects on handlers. Application code should not directly instantiate and use instances of :class:`Handler`. Instead, the :class:`Handler` class is a base class that @@ -918,16 +920,16 @@ Logged messages are formatted for presentation through instances of the use with the % operator and a dictionary. For formatting multiple messages in a batch, instances of -:class:`BufferingFormatter` can be used. In addition to the format string (which -is applied to each message in the batch), there is provision for header and -trailer format strings. +:class:`~handlers.BufferingFormatter` can be used. In addition to the format +string (which is applied to each message in the batch), there is provision for +header and trailer format strings. When filtering based on logger level and/or handler level is not enough, instances of :class:`Filter` can be added to both :class:`Logger` and -:class:`Handler` instances (through their :meth:`addFilter` method). Before -deciding to process a message further, both loggers and handlers consult all -their filters for permission. If any filter returns a false value, the message -is not processed further. +:class:`Handler` instances (through their :meth:`~Handler.addFilter` method). +Before deciding to process a message further, both loggers and handlers consult +all their filters for permission. If any filter returns a false value, the +message is not processed further. The basic :class:`Filter` functionality allows filtering by specific logger name. If this feature is used, messages sent to the named logger and its @@ -945,19 +947,20 @@ in production. This is so that errors which occur while handling logging events cause the application using logging to terminate prematurely. :class:`SystemExit` and :class:`KeyboardInterrupt` exceptions are never -swallowed. Other exceptions which occur during the :meth:`emit` method of a -:class:`Handler` subclass are passed to its :meth:`handleError` method. +swallowed. Other exceptions which occur during the :meth:`~Handler.emit` method +of a :class:`Handler` subclass are passed to its :meth:`~Handler.handleError` +method. -The default implementation of :meth:`handleError` in :class:`Handler` checks -to see if a module-level variable, :data:`raiseExceptions`, is set. If set, a -traceback is printed to :data:`sys.stderr`. If not set, the exception is swallowed. +The default implementation of :meth:`~Handler.handleError` in :class:`Handler` +checks to see if a module-level variable, :data:`raiseExceptions`, is set. If +set, a traceback is printed to :data:`sys.stderr`. If not set, the exception is +swallowed. .. note:: The default value of :data:`raiseExceptions` is ``True``. This is because during development, you typically want to be notified of any exceptions that occur. It's advised that you set :data:`raiseExceptions` to ``False`` for production usage. -.. currentmodule:: logging .. _arbitrary-object-messages: @@ -967,11 +970,11 @@ Using arbitrary objects as messages In the preceding sections and examples, it has been assumed that the message passed when logging the event is a string. However, this is not the only possibility. You can pass an arbitrary object as a message, and its -:meth:`__str__` method will be called when the logging system needs to convert -it to a string representation. In fact, if you want to, you can avoid +:meth:`~object.__str__` method will be called when the logging system needs to +convert it to a string representation. In fact, if you want to, you can avoid computing a string representation altogether - for example, the -:class:`SocketHandler` emits an event by pickling it and sending it over the -wire. +:class:`~handlers.SocketHandler` emits an event by pickling it and sending it +over the wire. Optimization @@ -980,9 +983,10 @@ Optimization Formatting of message arguments is deferred until it cannot be avoided. However, computing the arguments passed to the logging method can also be expensive, and you may want to avoid doing it if the logger will just throw -away your event. To decide what to do, you can call the :meth:`isEnabledFor` -method which takes a level argument and returns true if the event would be -created by the Logger for that level of call. You can write code like this:: +away your event. To decide what to do, you can call the +:meth:`~Logger.isEnabledFor` method which takes a level argument and returns +true if the event would be created by the Logger for that level of call. +You can write code like this:: if logger.isEnabledFor(logging.DEBUG): logger.debug('Message with %s, %s', expensive_func1(), diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 80326123d21..ee141f1a4b9 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -104,8 +104,9 @@ in :mod:`logging` itself) and defining handlers which are declared either in configurations. If no port is specified, the module's default :const:`DEFAULT_LOGGING_CONFIG_PORT` is used. Logging configurations will be sent as a file suitable for processing by :func:`fileConfig`. Returns a - :class:`Thread` instance on which you can call :meth:`start` to start the - server, and which you can :meth:`join` when appropriate. To stop the server, + :class:`~threading.Thread` instance on which you can call + :meth:`~threading.Thread.start` to start the server, and which you can + :meth:`~threading.Thread.join` when appropriate. To stop the server, call :func:`stopListening`. To send a configuration to the socket, read in the configuration file and @@ -169,11 +170,11 @@ otherwise, the context is used to determine what to instantiate. * *formatters* - the corresponding value will be a dict in which each key is a formatter id and each value is a dict describing how to - configure the corresponding Formatter instance. + configure the corresponding :class:`~logging.Formatter` instance. The configuring dict is searched for keys ``format`` and ``datefmt`` (with defaults of ``None``) and these are used to construct a - :class:`logging.Formatter` instance. + :class:`~logging.Formatter` instance. * *filters* - the corresponding value will be a dict in which each key is a filter id and each value is a dict describing how to configure @@ -711,8 +712,9 @@ format string, with a comma separator. An example time in ISO8601 format is The ``class`` entry is optional. It indicates the name of the formatter's class (as a dotted module and class name.) This option is useful for instantiating a -:class:`Formatter` subclass. Subclasses of :class:`Formatter` can present -exception tracebacks in an expanded or condensed format. +:class:`~logging.Formatter` subclass. Subclasses of +:class:`~logging.Formatter` can present exception tracebacks in an expanded or +condensed format. .. note:: Due to the use of :func:`eval` as described above, there are potential security risks which result from using the :func:`listen` to send diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index a771ca1ce3f..534c35c0e10 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -53,8 +53,8 @@ and :meth:`flush` methods). .. method:: flush() Flushes the stream by calling its :meth:`flush` method. Note that the - :meth:`close` method is inherited from :class:`Handler` and so does - no output, so an explicit :meth:`flush` call may be needed at times. + :meth:`close` method is inherited from :class:`~logging.Handler` and so + does no output, so an explicit :meth:`flush` call may be needed at times. .. _file-handler: @@ -142,8 +142,8 @@ new stream. This handler is not appropriate for use under Windows, because under Windows open log files cannot be moved or renamed - logging opens the files with exclusive locks - and so there is no need for such a handler. Furthermore, -*ST_INO* is not supported under Windows; :func:`stat` always returns zero for -this value. +*ST_INO* is not supported under Windows; :func:`~os.stat` always returns zero +for this value. .. class:: WatchedFileHandler(filename[,mode[, encoding[, delay]]]) @@ -305,7 +305,8 @@ sends logging output to a network socket. The base class uses a TCP socket. binary format. If there is an error with the socket, silently drops the packet. If the connection was previously lost, re-establishes the connection. To unpickle the record at the receiving end into a - :class:`LogRecord`, use the :func:`makeLogRecord` function. + :class:`~logging.LogRecord`, use the :func:`~logging.makeLogRecord` + function. .. method:: handleError() @@ -383,7 +384,8 @@ over UDP sockets. Pickles the record's attribute dictionary and writes it to the socket in binary format. If there is an error with the socket, silently drops the packet. To unpickle the record at the receiving end into a - :class:`LogRecord`, use the :func:`makeLogRecord` function. + :class:`~logging.LogRecord`, use the :func:`~logging.makeLogRecord` + function. .. method:: makeSocket() From 4f59ed7a9e31629dc6cd226907ff6bc269c2f52b Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 17 Aug 2013 15:43:51 +0300 Subject: [PATCH 292/480] =?UTF-8?q?#18705:=20fix=20a=20number=20of=20typos?= =?UTF-8?q?.=20=20Patch=20by=20F=C3=A9vry=20Thibault.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/Cookie.py | 2 +- Lib/_osx_support.py | 8 ++++---- Lib/_strptime.py | 2 +- Lib/aifc.py | 2 +- Lib/calendar.py | 2 +- Lib/compiler/pyassem.py | 2 +- Lib/difflib.py | 6 +++--- Lib/doctest.py | 4 ++-- Lib/genericpath.py | 2 +- Lib/inspect.py | 2 +- Lib/lib-tk/turtle.py | 8 ++++---- Lib/modulefinder.py | 2 +- Lib/multiprocessing/forking.py | 2 +- Lib/optparse.py | 2 +- Lib/pdb.py | 2 +- Lib/pickletools.py | 4 ++-- Lib/platform.py | 6 +++--- Lib/poplib.py | 2 +- Lib/rlcompleter.py | 2 +- Lib/subprocess.py | 2 +- Lib/tarfile.py | 2 +- Misc/ACKS | 1 + 22 files changed, 34 insertions(+), 33 deletions(-) diff --git a/Lib/Cookie.py b/Lib/Cookie.py index 2eda48c0bb1..db32980aca3 100644 --- a/Lib/Cookie.py +++ b/Lib/Cookie.py @@ -238,7 +238,7 @@ class CookieError(Exception): # a two-way quoting algorithm. Any non-text character is translated # into a 4 character sequence: a forward-slash followed by the # three-digit octal equivalent of the character. Any '\' or '"' is -# quoted with a preceeding '\' slash. +# quoted with a preceding '\' slash. # # These are taken from RFC2068 and RFC2109. # _LegalChars is the list of chars which don't require "'s diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 195561aff30..42a8ebb406d 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -152,7 +152,7 @@ def _find_appropriate_compiler(_config_vars): # are not installed. # # Futhermore, the compiler that can be used varies between - # Xcode releases. Upto Xcode 4 it was possible to use 'gcc-4.2' + # Xcode releases. Up to Xcode 4 it was possible to use 'gcc-4.2' # as the compiler, after that 'clang' should be used because # gcc-4.2 is either not present, or a copy of 'llvm-gcc' that # miscompiles Python. @@ -192,7 +192,7 @@ def _find_appropriate_compiler(_config_vars): if cc != oldcc: # Found a replacement compiler. - # Modify config vars using new compiler, if not already explictly + # Modify config vars using new compiler, if not already explicitly # overriden by an env variable, preserving additional arguments. for cv in _COMPILER_CONFIG_VARS: if cv in _config_vars and cv not in os.environ: @@ -274,7 +274,7 @@ def _check_for_unavailable_sdk(_config_vars): # compile an extension using an SDK that is not present # on the current machine it is better to not use an SDK # than to fail. This is particularly important with - # the standalong Command Line Tools alternative to a + # the standalone Command Line Tools alternative to a # full-blown Xcode install since the CLT packages do not # provide SDKs. If the SDK is not present, it is assumed # that the header files and dev libs have been installed @@ -378,7 +378,7 @@ def customize_config_vars(_config_vars): compilers are present, i.e. when installing pure Python dists. Customization of compiler paths and detection of unavailable archs is deferred - until the first extention module build is + until the first extension module build is requested (in distutils.sysconfig.customize_compiler). Currently called from distutils.sysconfig diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 2df30a22ea5..042db6f4f0e 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -222,7 +222,7 @@ def __seqToRE(self, to_convert, directive): """Convert a list to a regex string for matching a directive. Want possible matching values to be from longest to shortest. This - prevents the possibility of a match occuring for a value that also + prevents the possibility of a match occurring for a value that also a substring of a larger value that should have matched (e.g., 'abc' matching when 'abcdef' should have been the match). diff --git a/Lib/aifc.py b/Lib/aifc.py index a0cfe5fc57e..c2aad1ba66b 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -123,7 +123,7 @@ compression type, and then write audio frames using writeframesraw. When all frames have been written, either call writeframes('') or close() to patch up the sizes in the header. -Marks can be added anytime. If there are any marks, ypu must call +Marks can be added anytime. If there are any marks, you must call close() after all frames have been written. The close() method is called automatically when the class instance is destroyed. diff --git a/Lib/calendar.py b/Lib/calendar.py index 441b2f576bc..d3bd2362e93 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -220,7 +220,7 @@ def monthdayscalendar(self, year, month): def yeardatescalendar(self, year, width=3): """ Return the data for the specified year ready for formatting. The return - value is a list of month rows. Each month row contains upto width months. + value is a list of month rows. Each month row contains up to width months. Each month contains between 4 and 6 weeks and each week contains 1-7 days. Days are datetime.date objects. """ diff --git a/Lib/compiler/pyassem.py b/Lib/compiler/pyassem.py index 286be0c8c7d..f52f7d079f3 100644 --- a/Lib/compiler/pyassem.py +++ b/Lib/compiler/pyassem.py @@ -125,7 +125,7 @@ def order_blocks(start_block, exit_block): # Make sure every block appears in dominators, even if no # other block must precede it. dominators.setdefault(b, set()) - # preceeding blocks dominate following blocks + # preceding blocks dominate following blocks for c in b.get_followers(): while 1: dominators.setdefault(c, set()).add(b) diff --git a/Lib/difflib.py b/Lib/difflib.py index 3bbcb76b7ec..f94b3aca29b 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -586,7 +586,7 @@ def get_opcodes(self): def get_grouped_opcodes(self, n=3): """ Isolate change clusters by eliminating ranges with no changes. - Return a generator of groups with upto n lines of context. + Return a generator of groups with up to n lines of context. Each group is in the same format as returned by get_opcodes(). >>> from pprint import pprint @@ -1361,7 +1361,7 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None, linejunk -- passed on to ndiff (see ndiff documentation) charjunk -- passed on to ndiff (see ndiff documentation) - This function returns an interator which returns a tuple: + This function returns an iterator which returns a tuple: (from line tuple, to line tuple, boolean flag) from/to line tuple -- (line num, line text) @@ -1963,7 +1963,7 @@ def make_table(self,fromlines,tolines,fromdesc='',todesc='',context=False, self._make_prefix() # change tabs to spaces before it gets more difficult after we insert - # markkup + # markup fromlines,tolines = self._tab_newline_replace(fromlines,tolines) # create diffs iterator which generates side by side from/to data diff --git a/Lib/doctest.py b/Lib/doctest.py index 90bcca1fdbe..3e563c6a9cd 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -424,7 +424,7 @@ class Example: zero-based, with respect to the beginning of the DocTest. - indent: The example's indentation in the DocTest string. - I.e., the number of space characters that preceed the + I.e., the number of space characters that precede the example's first prompt. - options: A dictionary mapping from option flags to True or @@ -895,7 +895,7 @@ def find(self, obj, name=None, module=None, globs=None, extraglobs=None): if '__name__' not in globs: globs['__name__'] = '__main__' # provide a default module name - # Recursively expore `obj`, extracting DocTests. + # Recursively explore `obj`, extracting DocTests. tests = [] self._find(tests, obj, name, module, source_lines, globs, {}) # Sort the tests by alpha order of names, for consistency in diff --git a/Lib/genericpath.py b/Lib/genericpath.py index a0bf6013e93..7ddb94c08b9 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -22,7 +22,7 @@ def exists(path): # This follows symbolic links, so both islink() and isdir() can be true -# for the same path ono systems that support symlinks +# for the same path on systems that support symlinks def isfile(path): """Test whether a path is a regular file""" try: diff --git a/Lib/inspect.py b/Lib/inspect.py index 9e928b294f9..c2a32fee246 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -165,7 +165,7 @@ def isgenerator(object): """Return true if the object is a generator. Generator objects provide these attributes: - __iter__ defined to support interation over container + __iter__ defined to support iteration over container close raises a new GeneratorExit exception inside the generator to terminate the iteration gi_code code object diff --git a/Lib/lib-tk/turtle.py b/Lib/lib-tk/turtle.py index 29540518b74..f8188e1b707 100644 --- a/Lib/lib-tk/turtle.py +++ b/Lib/lib-tk/turtle.py @@ -1233,7 +1233,7 @@ def delay(self, delay=None): self._delayvalue = int(delay) def _incrementudc(self): - """Increment upadate counter.""" + """Increment update counter.""" if not TurtleScreen._RUNNING: TurtleScreen._RUNNNING = True raise Terminator @@ -2439,7 +2439,7 @@ def __init__(self, canvas=None, self.screen = TurtleScreen(canvas) RawTurtle.screens.append(self.screen) else: - raise TurtleGraphicsError("bad cavas argument %s" % canvas) + raise TurtleGraphicsError("bad canvas argument %s" % canvas) screen = self.screen TNavigator.__init__(self, screen.mode()) @@ -2684,7 +2684,7 @@ def shape(self, name=None): def shapesize(self, stretch_wid=None, stretch_len=None, outline=None): """Set/return turtle's stretchfactors/outline. Set resizemode to "user". - Optinonal arguments: + Optional arguments: stretch_wid : positive number stretch_len : positive number outline : positive number @@ -2975,7 +2975,7 @@ def clearstamps(self, n=None): def _goto(self, end): """Move the pen to the point end, thereby drawing a line - if pen is down. All other methodes for turtle movement depend + if pen is down. All other methods for turtle movement depend on this one. """ ## Version mit undo-stuff diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index 7f2bf8b1588..f6f84f3fdea 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -516,7 +516,7 @@ def report(self): # Print modules that may be missing, but then again, maybe not... if maybe: print - print "Submodules thay appear to be missing, but could also be", + print "Submodules that appear to be missing, but could also be", print "global names in the parent package:" for name in maybe: mods = self.badmodules[name].keys() diff --git a/Lib/multiprocessing/forking.py b/Lib/multiprocessing/forking.py index dc465b4ed25..6bddfb74ff5 100644 --- a/Lib/multiprocessing/forking.py +++ b/Lib/multiprocessing/forking.py @@ -367,7 +367,7 @@ def get_command_line(): def main(): ''' - Run code specifed by data received over pipe + Run code specified by data received over pipe ''' assert is_forking(sys.argv) diff --git a/Lib/optparse.py b/Lib/optparse.py index 731a2bb5edf..6c3f45651ff 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -1471,7 +1471,7 @@ def _match_long_opt(self, opt): """_match_long_opt(opt : string) -> string Determine which long option string 'opt' matches, ie. which one - it is an unambiguous abbrevation for. Raises BadOptionError if + it is an unambiguous abbreviation for. Raises BadOptionError if 'opt' doesn't unambiguously match any long option string. """ return _match_abbrev(opt, self._long_opt) diff --git a/Lib/pdb.py b/Lib/pdb.py index 5468d3fcaff..113b4e08927 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1095,7 +1095,7 @@ def help_exec(self): def help_run(self): print """run [args...] Restart the debugged python program. If a string is supplied, it is -splitted with "shlex" and the result is used as the new sys.argv. +split with "shlex" and the result is used as the new sys.argv. History, breakpoints, actions and debugger options are preserved. "restart" is an alias for "run".""" diff --git a/Lib/pickletools.py b/Lib/pickletools.py index d717728d417..8de53dd2507 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -804,7 +804,7 @@ def __repr__(self): obtype=StackObject, doc="""An object representing a contiguous slice of the stack. - This is used in conjuction with markobject, to represent all + This is used in conjunction with markobject, to represent all of the stack following the topmost markobject. For example, the POP_MARK opcode changes the stack from @@ -1929,7 +1929,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4): stack = [] # crude emulation of unpickler stack if memo is None: - memo = {} # crude emulation of unpicker memo + memo = {} # crude emulation of unpickler memo maxproto = -1 # max protocol number seen markstack = [] # bytecode positions of MARK opcodes indentchunk = ' ' * indentlevel diff --git a/Lib/platform.py b/Lib/platform.py index c0016a8198c..b6dab3b2349 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -228,7 +228,7 @@ def _dist_try_harder(distname,version,id): return 'OpenLinux',pkg[1],id if os.path.isdir('/usr/lib/setup'): - # Check for slackware verson tag file (thanks to Greg Andruk) + # Check for slackware version tag file (thanks to Greg Andruk) verfiles = os.listdir('/usr/lib/setup') for n in range(len(verfiles)-1, -1, -1): if verfiles[n][:14] != 'slack-version-': @@ -280,7 +280,7 @@ def _parse_release_file(firstline): if m is not None: return tuple(m.groups()) - # Unkown format... take the first two words + # Unknown format... take the first two words l = string.split(string.strip(firstline)) if l: version = l[0] @@ -800,7 +800,7 @@ def mac_ver(release='',versioninfo=('','',''),machine=''): versioninfo, machine) with versioninfo being a tuple (version, dev_stage, non_release_version). - Entries which cannot be determined are set to the paramter values + Entries which cannot be determined are set to the parameter values which default to ''. All tuple entries are strings. """ diff --git a/Lib/poplib.py b/Lib/poplib.py index e2b33ef10f4..dc7cbdf062e 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -321,7 +321,7 @@ class POP3_SSL(POP3): hostname - the hostname of the pop3 over ssl server port - port number - keyfile - PEM formatted file that countains your private key + keyfile - PEM formatted file that contains your private key certfile - PEM formatted certificate chain file See the methods of the parent class POP3 for more documentation. diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index 366c59c0cdf..6e4bd12ad31 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -116,7 +116,7 @@ def attr_matches(self, text): """Compute matches when text contains a dot. Assuming the text is of the form NAME.NAME....[NAME], and is - evaluatable in self.namespace, it will be evaluated and its attributes + evaluable in self.namespace, it will be evaluated and its attributes (as revealed by dir()) are used as possible completions. (For class instances, class members are also considered.) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 7fd1b025c28..50bec291c5a 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -143,7 +143,7 @@ class Popen(args, bufsize=0, executable=None, started to execute, will be re-raised in the parent. Additionally, the exception object will have one extra attribute called 'child_traceback', which is a string containing traceback information -from the childs point of view. +from the child's point of view. The most common exception raised is OSError. This occurs, for example, when trying to execute a non-existent file. Applications diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 16a6e86dce1..44ecd24eff1 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -330,7 +330,7 @@ class ExtractError(TarError): """General exception for extract errors.""" pass class ReadError(TarError): - """Exception for unreadble tar archives.""" + """Exception for unreadable tar archives.""" pass class CompressionError(TarError): """Exception for unavailable compression methods.""" diff --git a/Misc/ACKS b/Misc/ACKS index fae831e85cd..52019a827aa 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1015,6 +1015,7 @@ Mikhail Terekhov Victor Terrón Richard M. Tew Tobias Thelen +Févry Thibault Lowe Thiderman Nicolas M. Thiéry James Thomas From 8f061ed37ca88fe0d6b817d841f0df085a25e2f6 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 17 Aug 2013 15:01:54 +0200 Subject: [PATCH 293/480] Issue #18178: Fix ctypes on BSD. dlmalloc.c was compiled twice which broke malloc weak symbols. --- Misc/NEWS | 3 +++ Modules/_ctypes/libffi/fficonfig.py.in | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 37ada36c0c6..40816c702e3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,9 @@ Core and Builtins Library ------- +- Issue #18178: Fix ctypes on BSD. dlmalloc.c was compiled twice which broke + malloc weak symbols. + - Issue #18709: Fix CVE-2013-4238. The SSL module now handles NULL bytes inside subjectAltName correctly. Formerly the module has used OpenSSL's GENERAL_NAME_print() function to get the string represention of ASN.1 diff --git a/Modules/_ctypes/libffi/fficonfig.py.in b/Modules/_ctypes/libffi/fficonfig.py.in index 27c971f86f2..c9108148761 100644 --- a/Modules/_ctypes/libffi/fficonfig.py.in +++ b/Modules/_ctypes/libffi/fficonfig.py.in @@ -1,7 +1,6 @@ ffi_sources = """ src/prep_cif.c src/closures.c -src/dlmalloc.c """.split() ffi_platforms = { From c06449043d29c0a5b21e949e38df8ea614e1577b Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 17 Aug 2013 16:07:38 +0300 Subject: [PATCH 294/480] =?UTF-8?q?#18741:=20fix=20more=20typos.=20=20Patc?= =?UTF-8?q?h=20by=20F=C3=A9vry=20Thibault.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/ctypes/test/runtests.py | 2 +- Lib/ctypes/test/test_cfuncs.py | 2 +- Lib/ctypes/test/test_numbers.py | 2 +- Lib/ctypes/test/test_refcounts.py | 6 +++--- Lib/ctypes/test/test_structures.py | 2 +- Lib/distutils/command/sdist.py | 2 +- Lib/distutils/tests/test_build_clib.py | 2 +- Lib/idlelib/idle_test/mock_tk.py | 8 ++++---- Lib/idlelib/idle_test/test_formatparagraph.py | 2 +- Lib/lib2to3/fixes/fix_itertools.py | 4 ++-- Lib/lib2to3/fixes/fix_metaclass.py | 2 +- Lib/lib2to3/tests/test_parser.py | 2 +- Lib/sqlite3/test/regression.py | 6 ++++-- Lib/sqlite3/test/types.py | 2 +- Lib/test/test_support.py | 4 ++-- Lib/unittest/test/test_loader.py | 2 +- 16 files changed, 26 insertions(+), 24 deletions(-) diff --git a/Lib/ctypes/test/runtests.py b/Lib/ctypes/test/runtests.py index ec31fc831dd..b7a2b264721 100644 --- a/Lib/ctypes/test/runtests.py +++ b/Lib/ctypes/test/runtests.py @@ -2,7 +2,7 @@ Run all tests found in this directory, and print a summary of the results. Command line flags: - -q quiet mode: don't prnt anything while the tests are running + -q quiet mode: don't print anything while the tests are running -r run tests repeatedly, look for refcount leaks -u Add resources to the lits of allowed resources. '*' allows all diff --git a/Lib/ctypes/test/test_cfuncs.py b/Lib/ctypes/test/test_cfuncs.py index 493cbe979ce..598776e0080 100644 --- a/Lib/ctypes/test/test_cfuncs.py +++ b/Lib/ctypes/test/test_cfuncs.py @@ -188,7 +188,7 @@ def test_void(self): self.assertEqual(self._dll.tv_i(-42), None) self.assertEqual(self.S(), -42) -# The following repeates the above tests with stdcall functions (where +# The following repeats the above tests with stdcall functions (where # they are available) try: WinDLL diff --git a/Lib/ctypes/test/test_numbers.py b/Lib/ctypes/test/test_numbers.py index 79239068306..555e04a4a9b 100644 --- a/Lib/ctypes/test/test_numbers.py +++ b/Lib/ctypes/test/test_numbers.py @@ -212,7 +212,7 @@ def test_char_from_address(self): def test_init(self): # c_int() can be initialized from Python's int, and c_int. - # Not from c_long or so, which seems strange, abd should + # Not from c_long or so, which seems strange, abc should # probably be changed: self.assertRaises(TypeError, c_int, c_long(42)) diff --git a/Lib/ctypes/test/test_refcounts.py b/Lib/ctypes/test/test_refcounts.py index 35a81aa40be..fe4c4b92596 100644 --- a/Lib/ctypes/test/test_refcounts.py +++ b/Lib/ctypes/test/test_refcounts.py @@ -41,7 +41,7 @@ def func(*args): # this is the standard refcount for func self.assertEqual(grc(func), 2) - # the CFuncPtr instance holds atr least one refcount on func: + # the CFuncPtr instance holds at least one refcount on func: f = OtherCallback(func) self.assertTrue(grc(func) > 2) @@ -58,7 +58,7 @@ class X(ctypes.Structure): x = X() x.a = OtherCallback(func) - # the CFuncPtr instance holds atr least one refcount on func: + # the CFuncPtr instance holds at least one refcount on func: self.assertTrue(grc(func) > 2) # and may release it again @@ -71,7 +71,7 @@ class X(ctypes.Structure): f = OtherCallback(func) - # the CFuncPtr instance holds atr least one refcount on func: + # the CFuncPtr instance holds at least one refcount on func: self.assertTrue(grc(func) > 2) # create a cycle diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 2d7c816b241..a3350ce7774 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -108,7 +108,7 @@ class XX(Structure): def test_emtpy(self): # I had problems with these # - # Although these are patological cases: Empty Structures! + # Although these are pathological cases: Empty Structures! class X(Structure): _fields_ = [] diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py index d30de10673c..821420d62b8 100644 --- a/Lib/distutils/command/sdist.py +++ b/Lib/distutils/command/sdist.py @@ -183,7 +183,7 @@ def get_file_list(self): depends on the user's options. """ # new behavior when using a template: - # the file list is recalculated everytime because + # the file list is recalculated every time because # even if MANIFEST.in or setup.py are not changed # the user might have added some files in the tree that # need to be included. diff --git a/Lib/distutils/tests/test_build_clib.py b/Lib/distutils/tests/test_build_clib.py index bef1bd99536..12faa8a9548 100644 --- a/Lib/distutils/tests/test_build_clib.py +++ b/Lib/distutils/tests/test_build_clib.py @@ -77,7 +77,7 @@ def compile(*args, **kw): cmd.compiler = FakeCompiler() - # build_libraries is also doing a bit of typoe checking + # build_libraries is also doing a bit of typo checking lib = [('name', {'sources': 'notvalid'})] self.assertRaises(DistutilsSetupError, cmd.build_libraries, lib) diff --git a/Lib/idlelib/idle_test/mock_tk.py b/Lib/idlelib/idle_test/mock_tk.py index d2ab03801a4..9ce70905ed2 100644 --- a/Lib/idlelib/idle_test/mock_tk.py +++ b/Lib/idlelib/idle_test/mock_tk.py @@ -35,7 +35,7 @@ class Mbox(object): """Mock for tkinter.messagebox with an Mbox_func for each function. This module was 'tkMessageBox' in 2.x; hence the 'import as' in 3.x. - Example usage in test_module.py for testing functios in module.py: + Example usage in test_module.py for testing functions in module.py: --- from idlelib.idle_test.mock_tk import Mbox import module @@ -98,7 +98,7 @@ def _decode(self, index, endflag=0): This implements .index without converting the result back to a string. The result is contrained by the number of lines and linelengths of - self.data. For many indexes, the result is initally (1, 0). + self.data. For many indexes, the result is initially (1, 0). The input index may have any of several possible forms: * line.char float: converted to 'line.char' string; @@ -149,7 +149,7 @@ def _endex(self, endflag): -1: position before terminal \n; for .insert(), .delete 0: position after terminal \n; for .get, .delete index 1 - 1: same viewed as begininning of non-existent next line (for .index) + 1: same viewed as beginning of non-existent next line (for .index) ''' n = len(self.data) if endflag == 1: @@ -271,7 +271,7 @@ def see(self, index): "Scroll screen to make the character at INDEX is visible." pass - # The following is a Misc method inheritet by Text. + # The following is a Misc method inherited by Text. # It should properly go in a Misc mock, but is included here for now. def bind(sequence=None, func=None, add=None): diff --git a/Lib/idlelib/idle_test/test_formatparagraph.py b/Lib/idlelib/idle_test/test_formatparagraph.py index fca9540e92c..7d7affc81ce 100644 --- a/Lib/idlelib/idle_test/test_formatparagraph.py +++ b/Lib/idlelib/idle_test/test_formatparagraph.py @@ -244,7 +244,7 @@ class FormatEventTest(unittest.TestCase): """Test the formatting of text inside a Text widget. This is done with FormatParagraph.format.paragraph_event, - which calls funtions in the module as appropriate. + which calls functions in the module as appropriate. """ test_string = ( " '''this is a test of a reformat for a triple " diff --git a/Lib/lib2to3/fixes/fix_itertools.py b/Lib/lib2to3/fixes/fix_itertools.py index 27f8a49ec60..067641b8f86 100644 --- a/Lib/lib2to3/fixes/fix_itertools.py +++ b/Lib/lib2to3/fixes/fix_itertools.py @@ -34,8 +34,8 @@ def transform(self, node, results): # Remove the 'itertools' prefix = it.prefix it.remove() - # Replace the node wich contains ('.', 'function') with the - # function (to be consistant with the second part of the pattern) + # Replace the node which contains ('.', 'function') with the + # function (to be consistent with the second part of the pattern) dot.remove() func.parent.replace(func) diff --git a/Lib/lib2to3/fixes/fix_metaclass.py b/Lib/lib2to3/fixes/fix_metaclass.py index c86fbeaab2c..4f5593c5feb 100644 --- a/Lib/lib2to3/fixes/fix_metaclass.py +++ b/Lib/lib2to3/fixes/fix_metaclass.py @@ -71,7 +71,7 @@ def fixup_parse_tree(cls_node): def fixup_simple_stmt(parent, i, stmt_node): """ if there is a semi-colon all the parts count as part of the same simple_stmt. We just want the __metaclass__ part so we move - everything efter the semi-colon into its own simple_stmt node + everything after the semi-colon into its own simple_stmt node """ for semi_ind, node in enumerate(stmt_node.children): if node.type == token.SEMI: # *sigh* diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 26023813516..f66cb6b3f87 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -73,7 +73,7 @@ def test_3x_style_invalid_4(self): self.invalid_syntax("raise E from") -# Adapated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef +# Adaptated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef class TestFunctionAnnotations(GrammarTest): def test_1(self): self.validate("""def f(x) -> list: pass""") diff --git a/Lib/sqlite3/test/regression.py b/Lib/sqlite3/test/regression.py index 8a39d59071b..4c25f0005c8 100644 --- a/Lib/sqlite3/test/regression.py +++ b/Lib/sqlite3/test/regression.py @@ -159,7 +159,8 @@ def CheckSetIsolationLevel(self): def CheckCursorConstructorCallCheck(self): """ - Verifies that cursor methods check wether base class __init__ was called. + Verifies that cursor methods check whether base class __init__ was + called. """ class Cursor(sqlite.Cursor): def __init__(self, con): @@ -177,7 +178,8 @@ def __init__(self, con): def CheckConnectionConstructorCallCheck(self): """ - Verifies that connection methods check wether base class __init__ was called. + Verifies that connection methods check whether base class __init__ was + called. """ class Connection(sqlite.Connection): def __init__(self, name): diff --git a/Lib/sqlite3/test/types.py b/Lib/sqlite3/test/types.py index c5ab39bb771..400a4f24086 100644 --- a/Lib/sqlite3/test/types.py +++ b/Lib/sqlite3/test/types.py @@ -244,7 +244,7 @@ def CheckNumber1(self): self.assertEqual(type(value), float) def CheckNumber2(self): - """Checks wether converter names are cut off at '(' characters""" + """Checks whether converter names are cut off at '(' characters""" self.cur.execute("insert into test(n2) values (5)") value = self.cur.execute("select n2 from test").fetchone()[0] # if the converter is not used, it's an int instead of a float diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 1a1cac883c6..fe5a4f35e39 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -184,7 +184,7 @@ def unload(name): if sys.platform.startswith("win"): def _waitfor(func, pathname, waitall=False): - # Peform the operation + # Perform the operation func(pathname) # Now setup the wait loop if waitall: @@ -200,7 +200,7 @@ def _waitfor(func, pathname, waitall=False): # required when contention occurs. timeout = 0.001 while timeout < 1.0: - # Note we are only testing for the existance of the file(s) in + # Note we are only testing for the existence of the file(s) in # the contents of the directory regardless of any security or # access rights. If we have made it this far, we have sufficient # permissions to do that much using Python's equivalent of the diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py index 3544a20f7cf..d8d52f7c0df 100644 --- a/Lib/unittest/test/test_loader.py +++ b/Lib/unittest/test/test_loader.py @@ -324,7 +324,7 @@ def test_loadTestsFromName__relative_malformed_name(self): # Does loadTestsFromName raise TypeError when the `module` argument # isn't a module object? # - # XXX Accepts the not-a-module object, ignorning the object's type + # XXX Accepts the not-a-module object, ignoring the object's type # This should raise an exception or the method name should be changed # # XXX Some people are relying on this, so keep it for now From 74403cafc6b6bb24b67c4595be206a239bf67689 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 17 Aug 2013 16:56:09 +0300 Subject: [PATCH 295/480] =?UTF-8?q?#18466:=20fix=20more=20typos.=20=20Patc?= =?UTF-8?q?h=20by=20F=C3=A9vry=20Thibault.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Include/datetime.h | 2 +- Lib/collections.py | 2 +- Lib/test/test_ast.py | 2 +- Lib/test/test_cookielib.py | 2 +- Lib/test/test_datetime.py | 2 +- Lib/test/test_normalization.py | 2 +- Lib/test/test_urllib.py | 2 +- Misc/NEWS | 4 ++-- Modules/_ssl.c | 2 +- Modules/unicodedata.c | 2 +- PC/_subprocess.c | 2 +- PCbuild/readme.txt | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Include/datetime.h b/Include/datetime.h index 47abe5cb865..c0e7ffd00a9 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -42,7 +42,7 @@ typedef struct typedef struct { - PyObject_HEAD /* a pure abstract base clase */ + PyObject_HEAD /* a pure abstract base class */ } PyDateTime_TZInfo; diff --git a/Lib/collections.py b/Lib/collections.py index b9df2c14c91..0beb142be59 100644 --- a/Lib/collections.py +++ b/Lib/collections.py @@ -369,7 +369,7 @@ def namedtuple(typename, field_names, verbose=False, rename=False): result = namespace[typename] # For pickling to work, the __module__ variable needs to be set to the frame - # where the named tuple is created. Bypass this step in enviroments where + # where the named tuple is created. Bypass this step in environments where # sys._getframe is not defined (Jython for example) or sys._getframe is not # defined for arguments greater than 0 (IronPython). try: diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 4f80197f8d7..0a1ca4168a5 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -18,7 +18,7 @@ def to_tuple(t): # These tests are compiled through "exec" -# There should be atleast one test per statement +# There should be at least one test per statement exec_tests = [ # None "None", diff --git a/Lib/test/test_cookielib.py b/Lib/test/test_cookielib.py index dd0ad325d96..a3c31f67a38 100644 --- a/Lib/test/test_cookielib.py +++ b/Lib/test/test_cookielib.py @@ -329,7 +329,7 @@ class CookieTests(TestCase): ## commas and equals are commonly appear in the cookie value). This also ## means that if you fold multiple Set-Cookie header fields into one, ## comma-separated list, it'll be a headache to parse (at least my head -## starts hurting everytime I think of that code). +## starts hurting every time I think of that code). ## - Expires: You'll get all sorts of date formats in the expires, ## including emtpy expires attributes ("expires="). Be as flexible as you ## can, and certainly don't expect the weekday to be there; if you can't diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index d246d60797e..13ff5a38a18 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -124,7 +124,7 @@ def test_pickling_subclass(self): self.assertEqual(derived.tzname(None), 'cookie') ############################################################################# -# Base clase for testing a particular aspect of timedelta, time, date and +# Base class for testing a particular aspect of timedelta, time, date and # datetime comparisons. class HarmlessMixedComparison: diff --git a/Lib/test/test_normalization.py b/Lib/test/test_normalization.py index 3040a086187..1c45ad5c314 100644 --- a/Lib/test/test_normalization.py +++ b/Lib/test/test_normalization.py @@ -57,7 +57,7 @@ def test_main(self): c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]] except RangeError: # Skip unsupported characters; - # try atleast adding c1 if we are in part1 + # try at least adding c1 if we are in part1 if part == "@Part1": try: c1 = unistr(line.split(';')[0]) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index e60adb23357..3018ab198b4 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -812,7 +812,7 @@ def open_spam(self, url): # Everywhere else they work ok, but on those machines, sometimes # fail in one of the tests, sometimes in other. I have a linux, and # the tests go ok. -# If anybody has one of the problematic enviroments, please help! +# If anybody has one of the problematic environments, please help! # . Facundo # # def server(evt): diff --git a/Misc/NEWS b/Misc/NEWS index 40816c702e3..daa7e1fe7ea 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -3443,7 +3443,7 @@ Library - Issue #1285086: Speed up ``urllib.quote()`` and urllib.unquote for simple cases. -- Issue #8688: Distutils now recalculates MANIFEST everytime. +- Issue #8688: Distutils now recalculates MANIFEST every time. - Issue #5099: The ``__del__()`` method of ``subprocess.Popen`` (and the methods it calls) referenced global objects, causing errors to pop up during @@ -6299,7 +6299,7 @@ Library - Issue #3547: Fixed ctypes structures bitfields of varying integer sizes. -- Issue #3879: A regression in urllib.getproxies_enviroment was fixed. +- Issue #3879: A regression in urllib.getproxies_environment was fixed. - Issue #3863: Disabled a unit test of fork being called from a thread when running on platforms known to exhibit OS bugs when attempting that. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index b6c2f6b3223..3ff76a583c3 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1455,7 +1455,7 @@ static PyObject *PySSL_SSLshutdown(PySSLObject *self) * Otherwise OpenSSL might read in too much data, * eating clear text data that happens to be * transmitted after the SSL shutdown. - * Should be safe to call repeatedly everytime this + * Should be safe to call repeatedly every time this * function is used and the shutdown_seen_zero != 0 * condition is met. */ diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 95bdf3cfd1e..edc811e0811 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -520,7 +520,7 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) while(stackptr) { Py_UNICODE code = stack[--stackptr]; /* Hangul Decomposition adds three characters in - a single step, so we need atleast that much room. */ + a single step, so we need at least that much room. */ if (space < 3) { Py_ssize_t newsize = PyString_GET_SIZE(result) + 10; space += 10; diff --git a/PC/_subprocess.c b/PC/_subprocess.c index 689b0c8b433..195e343decb 100644 --- a/PC/_subprocess.c +++ b/PC/_subprocess.c @@ -331,7 +331,7 @@ getenvironment(PyObject* environment) PyObject* values; char* p; - /* convert environment dictionary to windows enviroment string */ + /* convert environment dictionary to windows environment string */ if (! PyMapping_Check(environment)) { PyErr_SetString( PyExc_TypeError, "environment must be dictionary or None"); diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 3395aac1bbd..00b50ece97d 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -262,7 +262,7 @@ SDK, in particular the 64-bit support. This includes an Itanium compiler In addition, you need the Visual Studio plugin for external C compilers, from http://sf.net/projects/vsextcomp. The plugin will wrap cl.exe, to locate the proper target compiler, and convert compiler options -accordingly. The project files require atleast version 0.9. +accordingly. The project files require at least version 0.9. Building for AMD64 ------------------ From 4c7ddcb349a8c75944c9b1f1419ac0ad103c3e66 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 17 Aug 2013 17:18:56 +0200 Subject: [PATCH 296/480] Issue #18768: coding style nitpick. Thanks to Vajrasky Kok --- Modules/_ssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 3ff76a583c3..10fb4029e3b 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -743,7 +743,7 @@ _get_peer_alt_names (X509 *certificate) { ASN1_STRING *as = NULL; name = sk_GENERAL_NAME_value(names, j); - gntype = name-> type; + gntype = name->type; switch (gntype) { case GEN_DIRNAME: /* we special-case DirName as a tuple of From dff9c14aff23d2fd6abb0fb0845f556d654faaf5 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 17 Aug 2013 17:25:18 +0200 Subject: [PATCH 297/480] Issue 18768: Correct doc string of RAND_edg(). Patch by Vajrasky Kok. --- Misc/NEWS | 2 ++ Modules/_ssl.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index daa7e1fe7ea..7947fc3a0c3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,8 @@ Core and Builtins Library ------- +- Issue 18768: Correct doc string of RAND_edg(). Patch by Vajrasky Kok. + - Issue #18178: Fix ctypes on BSD. dlmalloc.c was compiled twice which broke malloc weak symbols. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 10fb4029e3b..55c31555521 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1619,7 +1619,7 @@ PyDoc_STRVAR(PySSL_RAND_egd_doc, \n\ Queries the entropy gather daemon (EGD) on the socket named by 'path'.\n\ Returns number of bytes read. Raises SSLError if connection to EGD\n\ -fails or if it does provide enough data to seed PRNG."); +fails or if it does not provide enough data to seed PRNG."); #endif From 4a1943af07939a0598988475197c6bdf3d168ac8 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 18 Aug 2013 03:11:11 +0200 Subject: [PATCH 298/480] add missing # --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 7947fc3a0c3..600fc7702bc 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,7 +29,7 @@ Core and Builtins Library ------- -- Issue 18768: Correct doc string of RAND_edg(). Patch by Vajrasky Kok. +- Issue #18768: Correct doc string of RAND_edg(). Patch by Vajrasky Kok. - Issue #18178: Fix ctypes on BSD. dlmalloc.c was compiled twice which broke malloc weak symbols. From e8b62f328bbf017d377804d7cedb6e0d699d48ea Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 18 Aug 2013 18:22:34 -0400 Subject: [PATCH 299/480] Issue #18592: Add docstrings to file being tested (idlelib.SearchDialogBase.py). --- Lib/idlelib/SearchDialogBase.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/idlelib/SearchDialogBase.py b/Lib/idlelib/SearchDialogBase.py index f63e7ae37c7..bd1717a5af5 100644 --- a/Lib/idlelib/SearchDialogBase.py +++ b/Lib/idlelib/SearchDialogBase.py @@ -1,6 +1,23 @@ +'''Define SearchDialogBase used by Search, Replace, and Grep dialogs.''' from Tkinter import * class SearchDialogBase: + '''Create most of a modal search dialog (make_frame, create_widgets). + + The wide left column contains: + 1 or 2 text entry lines (create_entries, make_entry); + a row of standard radiobuttons (create_option_buttons); + a row of dialog specific radiobuttons (create_other_buttons). + + The narrow right column contains command buttons + (create_command_buttons, make_button). + These are bound to functions that execute the command. + + Except for command buttons, this base class is not limited to + items common to all three subclasses. Rather, it is the Find dialog + minus the "Find Next" command and its execution function. + The other dialogs override methods to replace and add widgets. + ''' title = "Search Dialog" icon = "Search" From 574c9100bca3cfdfbacb5f229ad17e837c386156 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 18 Aug 2013 18:27:02 -0400 Subject: [PATCH 300/480] Issue #18592: whitespace --- Lib/idlelib/SearchDialogBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/idlelib/SearchDialogBase.py b/Lib/idlelib/SearchDialogBase.py index bd1717a5af5..13ee90dfa4b 100644 --- a/Lib/idlelib/SearchDialogBase.py +++ b/Lib/idlelib/SearchDialogBase.py @@ -8,7 +8,7 @@ class SearchDialogBase: 1 or 2 text entry lines (create_entries, make_entry); a row of standard radiobuttons (create_option_buttons); a row of dialog specific radiobuttons (create_other_buttons). - + The narrow right column contains command buttons (create_command_buttons, make_button). These are bound to functions that execute the command. From be144210142d3bf0ca51348d0e808cdbdfcfd948 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 19 Aug 2013 01:05:09 -0400 Subject: [PATCH 301/480] Issue #18489: idlelib.SearchEngine - add docstrings (original patch by Phil Webster). --- Lib/idlelib/SearchEngine.py | 62 ++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/Lib/idlelib/SearchEngine.py b/Lib/idlelib/SearchEngine.py index 6745faff021..961a9218eee 100644 --- a/Lib/idlelib/SearchEngine.py +++ b/Lib/idlelib/SearchEngine.py @@ -1,19 +1,28 @@ +'''Define SearchEngine for search dialogs.''' import re from Tkinter import * import tkMessageBox def get(root): + '''Return the singleton SearchEngine instance for the process. + + The single SearchEngine saves settings between dialog instances. + If there is not a SearchEngine already, make one. + ''' if not hasattr(root, "_searchengine"): root._searchengine = SearchEngine(root) - # XXX This will never garbage-collect -- who cares + # This creates a cycle that persists until root is deleted. return root._searchengine class SearchEngine: + """Handles searching a text widget for Find, Replace, and Grep.""" def __init__(self, root): + '''Initialize Variables that save search state. + + The dialogs bind these to the UI elements present in the dialogs. + ''' self.root = root - # State shared by search, replace, and grep; - # the search dialogs bind these to UI elements. self.patvar = StringVar(root) # search pattern self.revar = BooleanVar(root) # regular expression? self.casevar = BooleanVar(root) # match case? @@ -56,6 +65,7 @@ def getcookedpat(self): return pat def getprog(self): + "Return compiled cooked search pattern." pat = self.getpat() if not pat: self.report_error(pat, "Empty regular expression") @@ -77,7 +87,7 @@ def getprog(self): return prog def report_error(self, pat, msg, col=-1): - # Derived class could overrid this with something fancier + # Derived class could override this with something fancier msg = "Error: " + str(msg) if pat: msg = msg + "\np\Pattern: " + str(pat) @@ -92,25 +102,23 @@ def setcookedpat(self, pat): self.setpat(pat) def search_text(self, text, prog=None, ok=0): - """Search a text widget for the pattern. + '''Return (lineno, matchobj) for prog in text widget, or None. - If prog is given, it should be the precompiled pattern. - Return a tuple (lineno, matchobj); None if not found. + If prog is given, it should be a precompiled pattern. + Wrap (yes/no) and direction (forward/back) settings are used. - This obeys the wrap and direction (back) settings. - - The search starts at the selection (if there is one) or - at the insert mark (otherwise). If the search is forward, - it starts at the right of the selection; for a backward - search, it starts at the left end. An empty match exactly - at either end of the selection (or at the insert mark if - there is no selection) is ignored unless the ok flag is true - -- this is done to guarantee progress. + The search starts at the selection (if there is one) or at the + insert mark (otherwise). If the search is forward, it starts + at the right of the selection; for a backward search, it + starts at the left end. An empty match exactly at either end + of the selection (or at the insert mark if there is no + selection) is ignored unless the ok flag is true -- this is + done to guarantee progress. If the search is allowed to wrap around, it will return the original selection if (and only if) it is the only match. + ''' - """ if not prog: prog = self.getprog() if not prog: @@ -179,10 +187,11 @@ def search_backward(self, text, prog, line, col, wrap, ok=0): col = len(chars) - 1 return None -# Helper to search backwards in a string. -# (Optimized for the case where the pattern isn't found.) - def search_reverse(prog, chars, col): + '''Search backwards in a string (line of text). + + This is done by searching forwards until there is no match. + ''' m = prog.search(chars) if not m: return None @@ -198,10 +207,9 @@ def search_reverse(prog, chars, col): i, j = m.span() return found -# Helper to get selection end points, defaulting to insert mark. -# Return a tuple of indices ("line.col" strings). - def get_selection(text): + '''Return tuple of 'line.col' indexes from selection or insert mark. + ''' try: first = text.index("sel.first") last = text.index("sel.last") @@ -213,8 +221,12 @@ def get_selection(text): last = first return first, last -# Helper to parse a text index into a (line, col) tuple. - def get_line_col(index): + '''Return (line, col) tuple of ints from 'line.col' string.''' line, col = map(int, index.split(".")) # Fails on invalid index return line, col + +##if __name__ == "__main__": +## from test import support; support.use_resources = ['gui'] +## import unittest +## unittest.main('idlelib.idle_test.test_searchengine', verbosity=2, exit=False) From b2f29f03d735fe1ba0fd4b61ff93d637b408138b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 19 Aug 2013 10:03:25 +0300 Subject: [PATCH 302/480] Issue #18761: Improved cross-references in email documentation. --- Doc/library/email.charset.rst | 2 +- Doc/library/email.errors.rst | 29 ++--- Doc/library/email.header.rst | 2 +- Doc/library/email.iterators.rst | 11 +- Doc/library/email.message.rst | 11 +- Doc/library/email.mime.rst | 29 +++-- Doc/library/email.parser.rst | 46 ++++---- Doc/library/email.rst | 193 ++++++++++++++++++-------------- Doc/library/email.util.rst | 14 ++- 9 files changed, 193 insertions(+), 144 deletions(-) diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst index 840aec6a14f..0ed6f0a8c1c 100644 --- a/Doc/library/email.charset.rst +++ b/Doc/library/email.charset.rst @@ -249,5 +249,5 @@ new entries to the global character set, alias, and codec registries: *charset* is the canonical name of a character set. *codecname* is the name of a Python codec, as appropriate for the second argument to the :func:`unicode` - built-in, or to the :meth:`encode` method of a Unicode string. + built-in, or to the :meth:`~unicode.encode` method of a Unicode string. diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 628fac92bdd..499d754fa08 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -25,7 +25,8 @@ The following exception classes are defined in the :mod:`email.errors` module: Raised under some error conditions when parsing the :rfc:`2822` headers of a message, this class is derived from :exc:`MessageParseError`. It can be raised - from the :meth:`Parser.parse` or :meth:`Parser.parsestr` methods. + from the :meth:`Parser.parse ` or + :meth:`Parser.parsestr ` methods. Situations where it can be raised include finding an envelope header after the first :rfc:`2822` header of the message, finding a continuation line before the @@ -37,7 +38,8 @@ The following exception classes are defined in the :mod:`email.errors` module: Raised under some error conditions when parsing the :rfc:`2822` headers of a message, this class is derived from :exc:`MessageParseError`. It can be raised - from the :meth:`Parser.parse` or :meth:`Parser.parsestr` methods. + from the :meth:`Parser.parse ` or + :meth:`Parser.parsestr ` methods. Situations where it can be raised include not being able to find the starting or terminating boundary in a :mimetype:`multipart/\*` message when strict parsing @@ -46,19 +48,20 @@ The following exception classes are defined in the :mod:`email.errors` module: .. exception:: MultipartConversionError() - Raised when a payload is added to a :class:`Message` object using - :meth:`add_payload`, but the payload is already a scalar and the message's - :mailheader:`Content-Type` main type is not either :mimetype:`multipart` or - missing. :exc:`MultipartConversionError` multiply inherits from - :exc:`MessageError` and the built-in :exc:`TypeError`. + Raised when a payload is added to a :class:`~email.message.Message` object + using :meth:`add_payload`, but the payload is already a scalar and the + message's :mailheader:`Content-Type` main type is not either + :mimetype:`multipart` or missing. :exc:`MultipartConversionError` multiply + inherits from :exc:`MessageError` and the built-in :exc:`TypeError`. - Since :meth:`Message.add_payload` is deprecated, this exception is rarely raised - in practice. However the exception may also be raised if the :meth:`attach` + Since :meth:`Message.add_payload` is deprecated, this exception is rarely + raised in practice. However the exception may also be raised if the + :meth:`~email.message.Message.attach` method is called on an instance of a class derived from :class:`~email.mime.nonmultipart.MIMENonMultipart` (e.g. :class:`~email.mime.image.MIMEImage`). -Here's the list of the defects that the :class:`~email.mime.parser.FeedParser` +Here's the list of the defects that the :class:`~email.parser.FeedParser` can find while parsing messages. Note that the defects are added to the message where the problem was found, so for example, if a message nested inside a :mimetype:`multipart/alternative` had a malformed header, that nested message @@ -86,7 +89,7 @@ this class is *not* an exception! or was otherwise malformed. * :class:`MultipartInvariantViolationDefect` -- A message claimed to be a - :mimetype:`multipart`, but no subparts were found. Note that when a message has - this defect, its :meth:`is_multipart` method may return false even though its - content type claims to be :mimetype:`multipart`. + :mimetype:`multipart`, but no subparts were found. Note that when a message + has this defect, its :meth:`~email.message.Message.is_multipart` method may + return false even though its content type claims to be :mimetype:`multipart`. diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index cb9cd1eb7af..4e585fce385 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -103,7 +103,7 @@ Here is the :class:`Header` class description: not provoke a :exc:`UnicodeError` is used. Optional *errors* is passed through to any :func:`unicode` or - :func:`ustr.encode` call, and defaults to "strict". + :meth:`unicode.encode` call, and defaults to "strict". .. method:: encode([splitchars]) diff --git a/Doc/library/email.iterators.rst b/Doc/library/email.iterators.rst index e1a35336c94..6621d51dcf7 100644 --- a/Doc/library/email.iterators.rst +++ b/Doc/library/email.iterators.rst @@ -6,8 +6,9 @@ Iterating over a message object tree is fairly easy with the -:meth:`Message.walk` method. The :mod:`email.iterators` module provides some -useful higher level iterations over message object trees. +:meth:`Message.walk ` method. The +:mod:`email.iterators` module provides some useful higher level iterations over +message object trees. .. function:: body_line_iterator(msg[, decode]) @@ -16,9 +17,11 @@ useful higher level iterations over message object trees. string payloads line-by-line. It skips over all the subpart headers, and it skips over any subpart with a payload that isn't a Python string. This is somewhat equivalent to reading the flat text representation of the message from - a file using :meth:`readline`, skipping over all the intervening headers. + a file using :meth:`~io.TextIOBase.readline`, skipping over all the + intervening headers. - Optional *decode* is passed through to :meth:`Message.get_payload`. + Optional *decode* is passed through to :meth:`Message.get_payload + `. .. function:: typed_subpart_iterator(msg[, maintype[, subtype]]) diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index da8e41c1a31..5076af62b59 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -48,8 +48,8 @@ Here are the methods of the :class:`Message` class: Note that this method is provided as a convenience and may not always format the message the way you want. For example, by default it mangles lines that begin with ``From``. For more flexibility, instantiate a - :class:`~email.generator.Generator` instance and use its :meth:`flatten` - method directly. For example:: + :class:`~email.generator.Generator` instance and use its + :meth:`~email.generator.Generator.flatten` method directly. For example:: from cStringIO import StringIO from email.generator import Generator @@ -494,8 +494,8 @@ Here are the methods of the :class:`Message` class: Set the ``boundary`` parameter of the :mailheader:`Content-Type` header to *boundary*. :meth:`set_boundary` will always quote *boundary* if - necessary. A :exc:`HeaderParseError` is raised if the message object has - no :mailheader:`Content-Type` header. + necessary. A :exc:`~email.errors.HeaderParseError` is raised if the + message object has no :mailheader:`Content-Type` header. Note that using this method is subtly different than deleting the old :mailheader:`Content-Type` header and adding a new one with the new @@ -589,7 +589,8 @@ Here are the methods of the :class:`Message` class: .. versionchanged:: 2.5 You do not need to set the epilogue to the empty string in order for the - :class:`Generator` to print a newline at the end of the file. + :class:`~email.generator.Generator` to print a newline at the end of the + file. .. attribute:: defects diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst index 957504da44d..dcf7b59c05e 100644 --- a/Doc/library/email.mime.rst +++ b/Doc/library/email.mime.rst @@ -35,7 +35,8 @@ Here are the classes: *_maintype* is the :mailheader:`Content-Type` major type (e.g. :mimetype:`text` or :mimetype:`image`), and *_subtype* is the :mailheader:`Content-Type` minor type (e.g. :mimetype:`plain` or :mimetype:`gif`). *_params* is a parameter - key/value dictionary and is passed directly to :meth:`Message.add_header`. + key/value dictionary and is passed directly to :meth:`Message.add_header + `. The :class:`MIMEBase` class always adds a :mailheader:`Content-Type` header (based on *_maintype*, *_subtype*, and *_params*), and a @@ -50,8 +51,9 @@ Here are the classes: A subclass of :class:`~email.mime.base.MIMEBase`, this is an intermediate base class for MIME messages that are not :mimetype:`multipart`. The primary - purpose of this class is to prevent the use of the :meth:`attach` method, - which only makes sense for :mimetype:`multipart` messages. If :meth:`attach` + purpose of this class is to prevent the use of the + :meth:`~email.message.Message.attach` method, which only makes sense for + :mimetype:`multipart` messages. If :meth:`~email.message.Message.attach` is called, a :exc:`~email.errors.MultipartConversionError` exception is raised. .. versionadded:: 2.2.2 @@ -76,7 +78,8 @@ Here are the classes: *_subparts* is a sequence of initial subparts for the payload. It must be possible to convert this sequence to a list. You can always attach new subparts - to the message by using the :meth:`Message.attach` method. + to the message by using the :meth:`Message.attach + ` method. Additional parameters for the :mailheader:`Content-Type` header are taken from the keyword arguments, or passed into the *_params* argument, which is a keyword @@ -99,8 +102,10 @@ Here are the classes: Optional *_encoder* is a callable (i.e. function) which will perform the actual encoding of the data for transport. This callable takes one argument, which is - the :class:`MIMEApplication` instance. It should use :meth:`get_payload` and - :meth:`set_payload` to change the payload to encoded form. It should also add + the :class:`MIMEApplication` instance. It should use + :meth:`~email.message.Message.get_payload` and + :meth:`~email.message.Message.set_payload` to change the payload to encoded + form. It should also add any :mailheader:`Content-Transfer-Encoding` or other headers to the message object as necessary. The default encoding is base64. See the :mod:`email.encoders` module for a list of the built-in encoders. @@ -127,8 +132,10 @@ Here are the classes: Optional *_encoder* is a callable (i.e. function) which will perform the actual encoding of the audio data for transport. This callable takes one argument, - which is the :class:`MIMEAudio` instance. It should use :meth:`get_payload` and - :meth:`set_payload` to change the payload to encoded form. It should also add + which is the :class:`MIMEAudio` instance. It should use + :meth:`~email.message.Message.get_payload` and + :meth:`~email.message.Message.set_payload` to change the payload to encoded + form. It should also add any :mailheader:`Content-Transfer-Encoding` or other headers to the message object as necessary. The default encoding is base64. See the :mod:`email.encoders` module for a list of the built-in encoders. @@ -153,8 +160,10 @@ Here are the classes: Optional *_encoder* is a callable (i.e. function) which will perform the actual encoding of the image data for transport. This callable takes one argument, - which is the :class:`MIMEImage` instance. It should use :meth:`get_payload` and - :meth:`set_payload` to change the payload to encoded form. It should also add + which is the :class:`MIMEImage` instance. It should use + :meth:`~email.message.Message.get_payload` and + :meth:`~email.message.Message.set_payload` to change the payload to encoded + form. It should also add any :mailheader:`Content-Transfer-Encoding` or other headers to the message object as necessary. The default encoding is base64. See the :mod:`email.encoders` module for a list of the built-in encoders. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index a91a7704243..6cc0494cde5 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -7,7 +7,8 @@ Message object structures can be created in one of two ways: they can be created from whole cloth by instantiating :class:`~email.message.Message` objects and -stringing them together via :meth:`attach` and :meth:`set_payload` calls, or they +stringing them together via :meth:`~email.message.Message.attach` and +:meth:`~email.message.Message.set_payload` calls, or they can be created by parsing a flat text representation of the email message. The :mod:`email` package provides a standard parser that understands most email @@ -16,8 +17,9 @@ or a file object, and the parser will return to you the root :class:`~email.message.Message` instance of the object structure. For simple, non-MIME messages the payload of this root object will likely be a string containing the text of the message. For MIME messages, the root object will -return ``True`` from its :meth:`is_multipart` method, and the subparts can be -accessed via the :meth:`get_payload` and :meth:`walk` methods. +return ``True`` from its :meth:`~email.message.Message.is_multipart` method, and +the subparts can be accessed via the :meth:`~email.message.Message.get_payload` +and :meth:`~email.message.Message.walk` methods. There are actually two parser interfaces available for use, the classic :class:`Parser` API and the incremental :class:`FeedParser` API. The classic @@ -127,7 +129,8 @@ class. Read all the data from the file-like object *fp*, parse the resulting text, and return the root message object. *fp* must support both the - :meth:`readline` and the :meth:`read` methods on file-like objects. + :meth:`~io.TextIOBase.readline` and the :meth:`~io.TextIOBase.read` + methods on file-like objects. The text contained in *fp* must be formatted as a block of :rfc:`2822` style headers and header continuation lines, optionally preceded by a @@ -147,7 +150,7 @@ class. Similar to the :meth:`parse` method, except it takes a string object instead of a file-like object. Calling this method on a string is exactly - equivalent to wrapping *text* in a :class:`StringIO` instance first and + equivalent to wrapping *text* in a :class:`~StringIO.StringIO` instance first and calling :meth:`parse`. Optional *headersonly* is as with the :meth:`parse` method. @@ -165,7 +168,7 @@ in the top-level :mod:`email` package namespace. Return a message object structure from a string. This is exactly equivalent to ``Parser().parsestr(s)``. Optional *_class* and *strict* are interpreted as - with the :class:`Parser` class constructor. + with the :class:``~email.parser.Parser` class constructor. .. versionchanged:: 2.2.2 The *strict* flag was added. @@ -175,7 +178,7 @@ in the top-level :mod:`email` package namespace. Return a message object structure tree from an open file object. This is exactly equivalent to ``Parser().parse(fp)``. Optional *_class* and *strict* - are interpreted as with the :class:`Parser` class constructor. + are interpreted as with the :class:``~email.parser.Parser` class constructor. .. versionchanged:: 2.2.2 The *strict* flag was added. @@ -193,32 +196,35 @@ Here are some notes on the parsing semantics: * Most non-\ :mimetype:`multipart` type messages are parsed as a single message object with a string payload. These objects will return ``False`` for - :meth:`is_multipart`. Their :meth:`get_payload` method will return a string - object. + :meth:`~email.message.Message.is_multipart`. Their + :meth:`~email.message.Message.get_payload` method will return a string object. * All :mimetype:`multipart` type messages will be parsed as a container message object with a list of sub-message objects for their payload. The outer - container message will return ``True`` for :meth:`is_multipart` and their - :meth:`get_payload` method will return the list of :class:`~email.message.Message` - subparts. + container message will return ``True`` for + :meth:`~email.message.Message.is_multipart` and their + :meth:`~email.message.Message.get_payload` method will return the list of + :class:`~email.message.Message` subparts. * Most messages with a content type of :mimetype:`message/\*` (e.g. :mimetype:`message/delivery-status` and :mimetype:`message/rfc822`) will also be parsed as container object containing a list payload of length 1. Their - :meth:`is_multipart` method will return ``True``. The single element in the - list payload will be a sub-message object. + :meth:`~email.message.Message.is_multipart` method will return ``True``. + The single element in the list payload will be a sub-message object. * Some non-standards compliant messages may not be internally consistent about their :mimetype:`multipart`\ -edness. Such messages may have a :mailheader:`Content-Type` header of type :mimetype:`multipart`, but their - :meth:`is_multipart` method may return ``False``. If such messages were parsed - with the :class:`FeedParser`, they will have an instance of the - :class:`MultipartInvariantViolationDefect` class in their *defects* attribute - list. See :mod:`email.errors` for details. + :meth:`~email.message.Message.is_multipart` method may return ``False``. + If such messages were parsed with the :class:`~email.parser.FeedParser`, + they will have an instance of the + :class:`~email.errors.MultipartInvariantViolationDefect` class in their + *defects* attribute list. See :mod:`email.errors` for details. .. rubric:: Footnotes .. [#] As of email package version 3.0, introduced in Python 2.4, the classic - :class:`Parser` was re-implemented in terms of the :class:`FeedParser`, so the - semantics and results are identical between the two parsers. + :class:`~email.parser.Parser` was re-implemented in terms of the + :class:`~email.parser.FeedParser`, so the semantics and results are + identical between the two parsers. diff --git a/Doc/library/email.rst b/Doc/library/email.rst index aaff1534ca4..4e90ce474f8 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -112,14 +112,15 @@ Here are the major differences between :mod:`email` version 4 and version 3: *Note that the version 3 names will continue to work until Python 2.6*. * The :mod:`email.mime.application` module was added, which contains the - :class:`MIMEApplication` class. + :class:`~email.mime.application.MIMEApplication` class. * Methods that were deprecated in version 3 have been removed. These include :meth:`Generator.__call__`, :meth:`Message.get_type`, :meth:`Message.get_main_type`, :meth:`Message.get_subtype`. * Fixes have been added for :rfc:`2231` support which can change some of the - return types for :func:`Message.get_param` and friends. Under some + return types for :func:`Message.get_param ` + and friends. Under some circumstances, values which used to return a 3-tuple now return simple strings (specifically, if all extended parameter segments were unencoded, there is no language and charset designation expected, so the return type is now a simple @@ -128,23 +129,24 @@ Here are the major differences between :mod:`email` version 4 and version 3: Here are the major differences between :mod:`email` version 3 and version 2: -* The :class:`FeedParser` class was introduced, and the :class:`Parser` class - was implemented in terms of the :class:`FeedParser`. All parsing therefore is +* The :class:`~email.parser.FeedParser` class was introduced, and the + :class:`~email.parser.Parser` class was implemented in terms of the + :class:`~email.parser.FeedParser`. All parsing therefore is non-strict, and parsing will make a best effort never to raise an exception. Problems found while parsing messages are stored in the message's *defect* attribute. * All aspects of the API which raised :exc:`DeprecationWarning`\ s in version 2 have been removed. These include the *_encoder* argument to the - :class:`MIMEText` constructor, the :meth:`Message.add_payload` method, the - :func:`Utils.dump_address_pair` function, and the functions :func:`Utils.decode` - and :func:`Utils.encode`. + :class:`~email.mime.text.MIMEText` constructor, the + :meth:`Message.add_payload` method, the :func:`Utils.dump_address_pair` + function, and the functions :func:`Utils.decode` and :func:`Utils.encode`. * New :exc:`DeprecationWarning`\ s have been added to: :meth:`Generator.__call__`, :meth:`Message.get_type`, :meth:`Message.get_main_type`, :meth:`Message.get_subtype`, and the *strict* - argument to the :class:`Parser` class. These are expected to be removed in - future versions. + argument to the :class:`~email.parser.Parser` class. These are expected to + be removed in future versions. * Support for Pythons earlier than 2.3 has been removed. @@ -152,53 +154,61 @@ Here are the differences between :mod:`email` version 2 and version 1: * The :mod:`email.Header` and :mod:`email.Charset` modules have been added. -* The pickle format for :class:`Message` instances has changed. Since this was - never (and still isn't) formally defined, this isn't considered a backward - incompatibility. However if your application pickles and unpickles - :class:`Message` instances, be aware that in :mod:`email` version 2, - :class:`Message` instances now have private variables *_charset* and - *_default_type*. +* The pickle format for :class:`~email.message.Message` instances has changed. + Since this was never (and still isn't) formally defined, this isn't + considered a backward incompatibility. However if your application pickles + and unpickles :class:`~email.message.Message` instances, be aware that in + :mod:`email` version 2, :class:`~email.message.Message` instances now have + private variables *_charset* and *_default_type*. -* Several methods in the :class:`Message` class have been deprecated, or their - signatures changed. Also, many new methods have been added. See the - documentation for the :class:`Message` class for details. The changes should be - completely backward compatible. +* Several methods in the :class:`~email.message.Message` class have been + deprecated, or their signatures changed. Also, many new methods have been + added. See the documentation for the :class:`~email.message.Message` class + for details. The changes should be completely backward compatible. * The object structure has changed in the face of :mimetype:`message/rfc822` - content types. In :mod:`email` version 1, such a type would be represented by a - scalar payload, i.e. the container message's :meth:`is_multipart` returned - false, :meth:`get_payload` was not a list object, but a single :class:`Message` - instance. + content types. In :mod:`email` version 1, such a type would be represented + by a scalar payload, i.e. the container message's + :meth:`~email.message.Message.is_multipart` returned false, + :meth:`~email.message.Message.get_payload` was not a list object, but a + single :class:`~email.message.Message` instance. This structure was inconsistent with the rest of the package, so the object representation for :mimetype:`message/rfc822` content types was changed. In :mod:`email` version 2, the container *does* return ``True`` from - :meth:`is_multipart`, and :meth:`get_payload` returns a list containing a single - :class:`Message` item. - - Note that this is one place that backward compatibility could not be completely - maintained. However, if you're already testing the return type of - :meth:`get_payload`, you should be fine. You just need to make sure your code - doesn't do a :meth:`set_payload` with a :class:`Message` instance on a container - with a content type of :mimetype:`message/rfc822`. - -* The :class:`Parser` constructor's *strict* argument was added, and its - :meth:`parse` and :meth:`parsestr` methods grew a *headersonly* argument. The - *strict* flag was also added to functions :func:`email.message_from_file` and - :func:`email.message_from_string`. - -* :meth:`Generator.__call__` is deprecated; use :meth:`Generator.flatten` - instead. The :class:`Generator` class has also grown the :meth:`clone` method. - -* The :class:`DecodedGenerator` class in the :mod:`email.Generator` module was - added. - -* The intermediate base classes :class:`MIMENonMultipart` and - :class:`MIMEMultipart` have been added, and interposed in the class hierarchy - for most of the other MIME-related derived classes. - -* The *_encoder* argument to the :class:`MIMEText` constructor has been - deprecated. Encoding now happens implicitly based on the *_charset* argument. + :meth:`~email.message.Message.is_multipart`, and + :meth:`~email.message.Message.get_payload` returns a list containing a single + :class:`~email.message.Message` item. + + Note that this is one place that backward compatibility could not be + completely maintained. However, if you're already testing the return type of + :meth:`~email.message.Message.get_payload`, you should be fine. You just need + to make sure your code doesn't do a :meth:`~email.message.Message.set_payload` + with a :class:`~email.message.Message` instance on a container with a content + type of :mimetype:`message/rfc822`. + +* The :class:`~email.parser.Parser` constructor's *strict* argument was added, + and its :meth:`~email.parser.Parser.parse` and + :meth:`~email.parser.Parser.parsestr` methods grew a *headersonly* argument. + The *strict* flag was also added to functions :func:`email.message_from_file` + and :func:`email.message_from_string`. + +* :meth:`Generator.__call__` is deprecated; use :meth:`Generator.flatten + ` instead. The + :class:`~email.generator.Generator` class has also grown the + :meth:`~email.generator.Generator.clone` method. + +* The :class:`~email.generator.DecodedGenerator` class in the + :mod:`email.generator` module was added. + +* The intermediate base classes + :class:`~email.mime.nonmultipart.MIMENonMultipart` and + :class:`~email.mime.multipart.MIMEMultipart` have been added, and interposed + in the class hierarchy for most of the other MIME-related derived classes. + +* The *_encoder* argument to the :class:`~email.mime.text.MIMEText` constructor + has been deprecated. Encoding now happens implicitly based on the + *_charset* argument. * The following functions in the :mod:`email.Utils` module have been deprecated: :func:`dump_address_pairs`, :func:`decode`, and :func:`encode`. The following @@ -231,17 +241,22 @@ package has the following differences: * :func:`messageFromFile` has been renamed to :func:`message_from_file`. -The :class:`Message` class has the following differences: +The :class:`~email.message.Message` class has the following differences: -* The method :meth:`asString` was renamed to :meth:`as_string`. +* The method :meth:`asString` was renamed to + :meth:`~email.message.Message.as_string`. -* The method :meth:`ismultipart` was renamed to :meth:`is_multipart`. +* The method :meth:`ismultipart` was renamed to + :meth:`~email.message.Message.is_multipart`. -* The :meth:`get_payload` method has grown a *decode* optional argument. +* The :meth:`~email.message.Message.get_payload` method has grown a *decode* + optional argument. -* The method :meth:`getall` was renamed to :meth:`get_all`. +* The method :meth:`getall` was renamed to + :meth:`~email.message.Message.get_all`. -* The method :meth:`addheader` was renamed to :meth:`add_header`. +* The method :meth:`addheader` was renamed to + :meth:`~email.message.Message.add_header`. * The method :meth:`gettype` was renamed to :meth:`get_type`. @@ -249,48 +264,57 @@ The :class:`Message` class has the following differences: * The method :meth:`getsubtype` was renamed to :meth:`get_subtype`. -* The method :meth:`getparams` was renamed to :meth:`get_params`. Also, whereas - :meth:`getparams` returned a list of strings, :meth:`get_params` returns a list - of 2-tuples, effectively the key/value pairs of the parameters, split on the - ``'='`` sign. +* The method :meth:`getparams` was renamed to + :meth:`~email.message.Message.get_params`. Also, whereas :meth:`getparams` + returned a list of strings, :meth:`~email.message.Message.get_params` returns + a list of 2-tuples, effectively the key/value pairs of the parameters, split + on the ``'='`` sign. -* The method :meth:`getparam` was renamed to :meth:`get_param`. +* The method :meth:`getparam` was renamed to + :meth:`~email.message.Message.get_param`. -* The method :meth:`getcharsets` was renamed to :meth:`get_charsets`. +* The method :meth:`getcharsets` was renamed to + :meth:`~email.message.Message.get_charsets`. -* The method :meth:`getfilename` was renamed to :meth:`get_filename`. +* The method :meth:`getfilename` was renamed to + :meth:`~email.message.Message.get_filename`. -* The method :meth:`getboundary` was renamed to :meth:`get_boundary`. +* The method :meth:`getboundary` was renamed to + :meth:`~email.message.Message.get_boundary`. -* The method :meth:`setboundary` was renamed to :meth:`set_boundary`. +* The method :meth:`setboundary` was renamed to + :meth:`~email.message.Message.set_boundary`. * The method :meth:`getdecodedpayload` was removed. To get similar - functionality, pass the value 1 to the *decode* flag of the get_payload() - method. + functionality, pass the value 1 to the *decode* flag of the + :meth:`~email.message.Message.get_payload` method. * The method :meth:`getpayloadastext` was removed. Similar functionality is - supported by the :class:`DecodedGenerator` class in the :mod:`email.generator` - module. + supported by the :class:`~email.generator.DecodedGenerator` class in the + :mod:`email.generator` module. * The method :meth:`getbodyastext` was removed. You can get similar - functionality by creating an iterator with :func:`typed_subpart_iterator` in the - :mod:`email.iterators` module. + functionality by creating an iterator with + :func:`~email.iterators.typed_subpart_iterator` in the :mod:`email.iterators` + module. -The :class:`Parser` class has no differences in its public interface. It does -have some additional smarts to recognize :mimetype:`message/delivery-status` -type messages, which it represents as a :class:`Message` instance containing -separate :class:`Message` subparts for each header block in the delivery status -notification [#]_. +The :class:`~email.parser.Parser` class has no differences in its public +interface. It does have some additional smarts to recognize +:mimetype:`message/delivery-status` type messages, which it represents as a +:class:`~email.message.Message` instance containing separate +:class:`~email.message.Message` subparts for each header block in the delivery +status notification [#]_. -The :class:`Generator` class has no differences in its public interface. There -is a new class in the :mod:`email.generator` module though, called -:class:`DecodedGenerator` which provides most of the functionality previously -available in the :meth:`Message.getpayloadastext` method. +The :class:`~email.generator.Generator` class has no differences in its public +interface. There is a new class in the :mod:`email.generator` module though, +called :class:`~email.generator.DecodedGenerator` which provides most of the +functionality previously available in the :meth:`Message.getpayloadastext` +method. The following modules and classes have been changed: -* The :class:`MIMEBase` class constructor arguments *_major* and *_minor* have - changed to *_maintype* and *_subtype* respectively. +* The :class:`~email.mime.base.MIMEBase` class constructor arguments *_major* + and *_minor* have changed to *_maintype* and *_subtype* respectively. * The ``Image`` class/module has been renamed to ``MIMEImage``. The *_minor* argument has been renamed to *_subtype*. @@ -303,7 +327,8 @@ The following modules and classes have been changed: but that clashed with the Python standard library module :mod:`rfc822` on some case-insensitive file systems. - Also, the :class:`MIMEMessage` class now represents any kind of MIME message + Also, the :class:`~email.mime.message.MIMEMessage` class now represents any + kind of MIME message with main type :mimetype:`message`. It takes an optional argument *_subtype* which is used to set the MIME subtype. *_subtype* defaults to :mimetype:`rfc822`. @@ -313,8 +338,8 @@ The following modules and classes have been changed: :mod:`email.utils` module. The ``MsgReader`` class/module has been removed. Its functionality is most -closely supported in the :func:`body_line_iterator` function in the -:mod:`email.iterators` module. +closely supported in the :func:`~email.iterators.body_line_iterator` function +in the :mod:`email.iterators` module. .. rubric:: Footnotes diff --git a/Doc/library/email.util.rst b/Doc/library/email.util.rst index 744f1da60bb..26f1919f527 100644 --- a/Doc/library/email.util.rst +++ b/Doc/library/email.util.rst @@ -41,8 +41,8 @@ There are several useful utilities provided in the :mod:`email.utils` module: This method returns a list of 2-tuples of the form returned by ``parseaddr()``. *fieldvalues* is a sequence of header field values as might be returned by - :meth:`Message.get_all`. Here's a simple example that gets all the recipients - of a message:: + :meth:`Message.get_all `. Here's a simple + example that gets all the recipients of a message:: from email.utils import getaddresses @@ -130,7 +130,8 @@ There are several useful utilities provided in the :mod:`email.utils` module: .. function:: collapse_rfc2231_value(value[, errors[, fallback_charset]]) When a header parameter is encoded in :rfc:`2231` format, - :meth:`Message.get_param` may return a 3-tuple containing the character set, + :meth:`Message.get_param ` may return a + 3-tuple containing the character set, language, and value. :func:`collapse_rfc2231_value` turns this into a unicode string. Optional *errors* is passed to the *errors* argument of the built-in :func:`unicode` function; it defaults to ``replace``. Optional @@ -152,11 +153,12 @@ There are several useful utilities provided in the :mod:`email.utils` module: .. versionchanged:: 2.4 The :func:`decode` function has been removed; use the - :meth:`Header.decode_header` method instead. + :meth:`Header.decode_header ` method + instead. .. versionchanged:: 2.4 - The :func:`encode` function has been removed; use the :meth:`Header.encode` - method instead. + The :func:`encode` function has been removed; use the :meth:`Header.encode + ` method instead. .. rubric:: Footnotes From 6d9112c386b6bcfaef22bdbb511e985765d015a8 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 19 Aug 2013 17:36:29 +0200 Subject: [PATCH 303/480] Issue #18777: The ssl module now uses the new CRYPTO_THREADID API of OpenSSL 1.0.0+ instead of the deprecated CRYPTO id callback function. --- Misc/NEWS | 3 +++ Modules/_ssl.c | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 600fc7702bc..0f4b701e208 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,9 @@ Core and Builtins Library ------- +- Issue #18777: The ssl module now uses the new CRYPTO_THREADID API of + OpenSSL 1.0.0+ instead of the deprecated CRYPTO id callback function. + - Issue #18768: Correct doc string of RAND_edg(). Patch by Vajrasky Kok. - Issue #18178: Fix ctypes on BSD. dlmalloc.c was compiled twice which broke diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 55c31555521..b3eb6ec339b 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1649,9 +1649,21 @@ static PyMethodDef PySSL_methods[] = { static PyThread_type_lock *_ssl_locks = NULL; -static unsigned long _ssl_thread_id_function (void) { +#if OPENSSL_VERSION_NUMBER >= 0x10000000 +/* use new CRYPTO_THREADID API. */ +static void +_ssl_threadid_callback(CRYPTO_THREADID *id) +{ + CRYPTO_THREADID_set_numeric(id, + (unsigned long)PyThread_get_thread_ident()); +} +#else +/* deprecated CRYPTO_set_id_callback() API. */ +static unsigned long +_ssl_thread_id_function (void) { return PyThread_get_thread_ident(); } +#endif static void _ssl_thread_locking_function (int mode, int n, const char *file, int line) { /* this function is needed to perform locking on shared data @@ -1702,7 +1714,11 @@ static int _setup_ssl_threads(void) { } } CRYPTO_set_locking_callback(_ssl_thread_locking_function); +#if OPENSSL_VERSION_NUMBER >= 0x10000000 + CRYPTO_THREADID_set_callback(_ssl_threadid_callback); +#else CRYPTO_set_id_callback(_ssl_thread_id_function); +#endif } return 1; } From eb2eea2106fc7580a3f2ddbf9abc6ee7920ccb83 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 19 Aug 2013 22:53:46 +0300 Subject: [PATCH 304/480] Issue #18647: Correctly bound calculated min/max width of a subexpression. Now max width is MAXREPEAT on 32- and 64-bit platforms when one of subexpressions is unbounded repetition. --- Lib/sre_parse.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index 75f8c9601df..e8d35a6d412 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -142,12 +142,12 @@ def getwidth(self): # determine the width (min, max) for this subpattern if self.width: return self.width - lo = hi = 0L + lo = hi = 0 UNITCODES = (ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY) REPEATCODES = (MIN_REPEAT, MAX_REPEAT) for op, av in self.data: if op is BRANCH: - i = sys.maxint + i = MAXREPEAT - 1 j = 0 for av in av[1]: l, h = av.getwidth() @@ -165,14 +165,14 @@ def getwidth(self): hi = hi + j elif op in REPEATCODES: i, j = av[2].getwidth() - lo = lo + long(i) * av[0] - hi = hi + long(j) * av[1] + lo = lo + i * av[0] + hi = hi + j * av[1] elif op in UNITCODES: lo = lo + 1 hi = hi + 1 elif op == SUCCESS: break - self.width = int(min(lo, sys.maxint)), int(min(hi, sys.maxint)) + self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT) return self.width class Tokenizer: From 940d4c7c5cef6f9aa2194078b6be177885606fe7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 19 Aug 2013 23:04:52 +0300 Subject: [PATCH 305/480] Issue #18647: A regular expression in the doctest module rewritten so that determined minimal width of repeated subexpression is >0 (an empty line was not matched in any case). --- Lib/doctest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/doctest.py b/Lib/doctest.py index 3e563c6a9cd..efa07d7ea51 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -564,7 +564,7 @@ class DocTestParser: # Want consists of any non-blank lines that do not start with PS1. (?P (?:(?![ ]*$) # Not a blank line (?![ ]*>>>) # Not a line starting with PS1 - .*$\n? # But any other line + .+$\n? # But any other line )*) ''', re.MULTILINE | re.VERBOSE) From 6ae24a46df1b4d169825aee3e82d1de72d9b82e7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 19 Aug 2013 23:20:07 +0300 Subject: [PATCH 306/480] Issue #2537: Remove breaked check which prevented valid regular expressions. Patch by Meador Inge. See also issue #18647. --- Lib/sre_compile.py | 2 -- Lib/test/test_re.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py index 425a1f80cb2..15d23243c16 100644 --- a/Lib/sre_compile.py +++ b/Lib/sre_compile.py @@ -355,8 +355,6 @@ def _optimize_unicode(charset, fixup): def _simple(av): # check if av is a "simple" operator lo, hi = av[2].getwidth() - #if lo == 0 and hi == MAXREPEAT: - # raise error, "nothing to repeat" return lo == hi == 1 and av[2][0][0] != SUBPATTERN def _compile_info(code, pattern, flags): diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 2be5f5cb2ea..8b277cfc02e 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -908,6 +908,16 @@ def test_issue17998(self): [b'xyz'], msg=pattern) + def test_bug_2537(self): + # issue 2537: empty submatches + for outer_op in ('{0,}', '*', '+', '{1,187}'): + for inner_op in ('{0,}', '*', '?'): + r = re.compile("^((x|y)%s)%s" % (inner_op, outer_op)) + m = r.match("xyyzy") + self.assertEqual(m.group(0), "xyy") + self.assertEqual(m.group(1), "") + self.assertEqual(m.group(2), "y") + def run_re_tests(): from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR if verbose: From fc6a1bcb5e70a4b8df07f991ea245122273dcdf2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 20 Aug 2013 20:08:53 +0300 Subject: [PATCH 307/480] Issue #13461: Fix a crash in the TextIOWrapper.tell method and in the "replace" error handler on 64-bit platforms. Patch by Yogesh Chaudhari. --- Misc/ACKS | 1 + Misc/NEWS | 6 ++++++ Modules/_io/textio.c | 2 +- Python/codecs.c | 2 +- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS index 52019a827aa..8aba36e3005 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -170,6 +170,7 @@ Jeffrey Chang Mitch Chapman Greg Chapman Brad Chapman +Yogesh Chaudhari David Chaum Nicolas Chauvat Michael Chermside diff --git a/Misc/NEWS b/Misc/NEWS index 0f4b701e208..9ee4cfe2670 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -9,6 +9,9 @@ What's New in Python 2.7.6? Core and Builtins ----------------- +- Issue #13461: Fix a crash in the "replace" error handler on 64-bit platforms. + Patch by Yogesh Chaudhari. + - Issue #15866: The xmlcharrefreplace error handler no more produces two XML entities for a non-BMP character on narrow build. @@ -29,6 +32,9 @@ Core and Builtins Library ------- +- Issue #13461: Fix a crash in the TextIOWrapper.tell method on 64-bit + platforms. Patch by Yogesh Chaudhari. + - Issue #18777: The ssl module now uses the new CRYPTO_THREADID API of OpenSSL 1.0.0+ instead of the deprecated CRYPTO id callback function. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index cd6d443e23f..680275876e5 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2271,7 +2271,7 @@ textiowrapper_tell(textio *self, PyObject *args) int dec_flags; PyObject *decoded = PyObject_CallMethod( - self->decoder, "decode", "s#", input, 1); + self->decoder, "decode", "s#", input, (Py_ssize_t)1); if (check_decoded(decoded) < 0) goto fail; chars_decoded += PyUnicode_GET_SIZE(decoded); diff --git a/Python/codecs.c b/Python/codecs.c index 91147a07a32..69498c4b0c9 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -521,7 +521,7 @@ PyObject *PyCodec_ReplaceErrors(PyObject *exc) Py_UNICODE res = Py_UNICODE_REPLACEMENT_CHARACTER; if (PyUnicodeDecodeError_GetEnd(exc, &end)) return NULL; - return Py_BuildValue("(u#n)", &res, 1, end); + return Py_BuildValue("(u#n)", &res, (Py_ssize_t)1, end); } else if (PyObject_IsInstance(exc, PyExc_UnicodeTranslateError)) { PyObject *res; From b6a6fc5a3de1034c4b00210b81e70feb012c7da4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 20 Aug 2013 20:38:21 +0300 Subject: [PATCH 308/480] Issue #8865: Concurrent invocation of select.poll.poll() now raises a RuntimeError exception. Patch by Christian Schubert. --- Lib/test/test_poll.py | 42 ++++++++++++++++++++++++++++++++++++++++-- Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/selectmodule.c | 13 +++++++++++++ 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index 55294f8c229..219fa0e4647 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -1,8 +1,16 @@ # Test case for the os.poll() function -import os, select, random, unittest +import os +import random +import select import _testcapi -from test.test_support import TESTFN, run_unittest +try: + import threading +except ImportError: + threading = None +import time +import unittest +from test.test_support import TESTFN, run_unittest, reap_threads try: select.poll @@ -160,6 +168,36 @@ def test_poll3(self): self.assertRaises(OverflowError, pollster.poll, _testcapi.INT_MAX + 1) self.assertRaises(OverflowError, pollster.poll, _testcapi.UINT_MAX + 1) + @unittest.skipUnless(threading, 'Threading required for this test.') + @reap_threads + def test_threaded_poll(self): + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + rfds = [] + for i in range(10): + fd = os.dup(r) + self.addCleanup(os.close, fd) + rfds.append(fd) + pollster = select.poll() + for fd in rfds: + pollster.register(fd, select.POLLIN) + + t = threading.Thread(target=pollster.poll) + t.start() + try: + time.sleep(0.5) + # trigger ufds array reallocation + for fd in rfds: + pollster.unregister(fd) + pollster.register(w, select.POLLOUT) + self.assertRaises(RuntimeError, pollster.poll) + finally: + # and make the call to poll() from the thread return + os.write(w, b'spam') + t.join() + + def test_main(): run_unittest(PollTests) diff --git a/Misc/ACKS b/Misc/ACKS index 8aba36e3005..c6c2e2f2396 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -912,6 +912,7 @@ Peter Schneider-Kamp Arvin Schnell Scott Schram Chad J. Schroeder +Christian Schubert Sam Schulenburg Stefan Schwarzer Dietmar Schwertberger diff --git a/Misc/NEWS b/Misc/NEWS index 9ee4cfe2670..4c39da3748f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #8865: Concurrent invocation of select.poll.poll() now raises a + RuntimeError exception. Patch by Christian Schubert. + - Issue #13461: Fix a crash in the TextIOWrapper.tell method on 64-bit platforms. Patch by Yogesh Chaudhari. diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 4cbf885b18f..00d5f6c058c 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -318,6 +318,7 @@ typedef struct { int ufd_uptodate; int ufd_len; struct pollfd *ufds; + int poll_running; } pollObject; static PyTypeObject poll_Type; @@ -513,16 +514,27 @@ poll_poll(pollObject *self, PyObject *args) return NULL; } + /* Avoid concurrent poll() invocation, issue 8865 */ + if (self->poll_running) { + PyErr_SetString(PyExc_RuntimeError, + "concurrent poll() invocation"); + return NULL; + } + /* Ensure the ufd array is up to date */ if (!self->ufd_uptodate) if (update_ufd_array(self) == 0) return NULL; + self->poll_running = 1; + /* call poll() */ Py_BEGIN_ALLOW_THREADS poll_result = poll(self->ufds, self->ufd_len, timeout); Py_END_ALLOW_THREADS + self->poll_running = 0; + if (poll_result < 0) { PyErr_SetFromErrno(SelectError); return NULL; @@ -599,6 +611,7 @@ newPollObject(void) array pointed to by ufds matches the contents of the dictionary. */ self->ufd_uptodate = 0; self->ufds = NULL; + self->poll_running = 0; self->dict = PyDict_New(); if (self->dict == NULL) { Py_DECREF(self); From 6a44f925680065ded5f8eba5b0767671db0978e4 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 20 Aug 2013 20:35:20 -0400 Subject: [PATCH 309/480] Fix UnboundLocalError regression due to previous incorrect fix for issue 16248. --- Lib/lib-tk/Tkinter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/lib-tk/Tkinter.py b/Lib/lib-tk/Tkinter.py index ea563313f1d..b5875e58dcc 100644 --- a/Lib/lib-tk/Tkinter.py +++ b/Lib/lib-tk/Tkinter.py @@ -1634,7 +1634,7 @@ def __init__(self, screenName=None, baseName=None, className='Tk', # ensure that self.tk is always _something_. self.tk = None if baseName is None: - import sys, os + import os baseName = os.path.basename(sys.argv[0]) baseName, ext = os.path.splitext(baseName) if ext not in ('.py', '.pyc', '.pyo'): From 302121956133ce60db3ea00b57e11be2b776d20c Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 21 Aug 2013 13:26:05 +0200 Subject: [PATCH 310/480] Issue #18747: Re-seed OpenSSL's pseudo-random number generator after fork. A pthread_atfork() child handler is used to seeded the PRNG with pid, time and some stack data. --- Misc/NEWS | 4 +++ Modules/_ssl.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ configure | 11 ++++++++ configure.ac | 1 + pyconfig.h.in | 3 +++ 5 files changed, 91 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 4c39da3748f..04317f83999 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,10 @@ Core and Builtins Library ------- +- Issue #18747: Re-seed OpenSSL's pseudo-random number generator after fork. + A pthread_atfork() child handler is used to seeded the PRNG with pid, time + and some stack data. + - Issue #8865: Concurrent invocation of select.poll.poll() now raises a RuntimeError exception. Patch by Christian Schubert. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index b3eb6ec339b..1c769edb091 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -18,6 +18,11 @@ #ifdef WITH_THREAD #include "pythread.h" + +#ifdef HAVE_PTHREAD_ATFORK +# include +#endif + #define PySSL_BEGIN_ALLOW_THREADS { \ PyThreadState *_save = NULL; \ if (_ssl_locks_count>0) {_save = PyEval_SaveThread();} @@ -1621,7 +1626,69 @@ Queries the entropy gather daemon (EGD) on the socket named by 'path'.\n\ Returns number of bytes read. Raises SSLError if connection to EGD\n\ fails or if it does not provide enough data to seed PRNG."); +/* Seed OpenSSL's PRNG at fork(), http://bugs.python.org/issue18747 + * + * The child handler seeds the PRNG from pseudo-random data like pid, the + * current time (nanoseconds, miliseconds or seconds) and an uninitialized + * array. The array contains stack variables that are impossible to predict + * on most systems, e.g. function return address (subject to ASLR), the + * stack protection canary and automatic variables. + * The code is inspired by Apache's ssl_rand_seed() function. + * + * Note: + * The code uses pthread_atfork() until Python has a proper atfork API. The + * handlers are not removed from the child process. + */ + +#if defined(HAVE_PTHREAD_ATFORK) && defined(WITH_THREAD) +#define PYSSL_RAND_ATFORK 1 + +static void +PySSL_RAND_atfork_child(void) +{ + struct { + char stack[128]; /* uninitialized (!) stack data, 128 is an + arbitrary number. */ + pid_t pid; /* current pid */ + time_t time; /* current time */ + } seed; + +#ifdef WITH_VALGRIND + VALGRIND_MAKE_MEM_DEFINED(seed.stack, sizeof(seed.stack)); #endif + seed.pid = getpid(); + seed.time = time(NULL); + +#if 0 + fprintf(stderr, "PySSL_RAND_atfork_child() seeds %i bytes in pid %i\n", + (int)sizeof(seed), seed.pid); +#endif + RAND_add((unsigned char *)&seed, sizeof(seed), 0.0); +} + +static int +PySSL_RAND_atfork(void) +{ + static int registered = 0; + int retval; + + if (registered) + return 0; + + retval = pthread_atfork(NULL, /* prepare */ + NULL, /* parent */ + PySSL_RAND_atfork_child); /* child */ + if (retval != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + registered = 1; + return 0; +} +#endif /* HAVE_PTHREAD_ATFORK */ + +#endif /* HAVE_OPENSSL_RAND */ + /* List of functions exported by this module. */ @@ -1833,4 +1900,9 @@ init_ssl(void) r = PyString_FromString(SSLeay_version(SSLEAY_VERSION)); if (r == NULL || PyModule_AddObject(m, "OPENSSL_VERSION", r)) return; + +#ifdef PYSSL_RAND_ATFORK + if (PySSL_RAND_atfork() == -1) + return; +#endif } diff --git a/configure b/configure index 5f376af7756..68e47842b56 100755 --- a/configure +++ b/configure @@ -9627,6 +9627,17 @@ $as_echo "#define HAVE_BROKEN_PTHREAD_SIGMASK 1" >>confdefs.h ;; esac +fi +done + + for ac_func in pthread_atfork +do : + ac_fn_c_check_func "$LINENO" "pthread_atfork" "ac_cv_func_pthread_atfork" +if test "x$ac_cv_func_pthread_atfork" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_PTHREAD_ATFORK 1 +_ACEOF + fi done diff --git a/configure.ac b/configure.ac index b4ed0107a83..d922bc7cb17 100644 --- a/configure.ac +++ b/configure.ac @@ -2569,6 +2569,7 @@ if test "$posix_threads" = "yes"; then [Define if pthread_sigmask() does not work on your system.]) ;; esac]) + AC_CHECK_FUNCS(pthread_atfork) fi diff --git a/pyconfig.h.in b/pyconfig.h.in index 65df68aed9f..8ac017e2383 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -520,6 +520,9 @@ /* Define if you have GNU PTH threads. */ #undef HAVE_PTH +/* Define to 1 if you have the `pthread_atfork' function. */ +#undef HAVE_PTHREAD_ATFORK + /* Defined for Solaris 2.6 bug in pthread header. */ #undef HAVE_PTHREAD_DESTRUCTOR From 0e6fa41ae0db2dcd62ab031781e96084ed305ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Wed, 21 Aug 2013 18:25:00 +0200 Subject: [PATCH 311/480] Issue #15233: Python now guarantees that callables registered with the atexit module will be called in a deterministic order. --- Doc/library/atexit.rst | 15 ++++++++------- Misc/NEWS | 3 +++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 37d8d500ef1..0b5e121fe63 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -15,13 +15,14 @@ The :mod:`atexit` module defines a single function to register cleanup functions. Functions thus registered are automatically executed upon normal -interpreter termination. The order in which the functions are called is not -defined; if you have cleanup operations that depend on each other, you should -wrap them in a function and register that one. This keeps :mod:`atexit` simple. - -Note: the functions registered via this module are not called when the program -is killed by a signal not handled by Python, when a Python fatal internal error -is detected, or when :func:`os._exit` is called. +interpreter termination. :mod:`atexit` runs these functions in the *reverse* +order in which they were registered; if you register ``A``, ``B``, and ``C``, +at interpreter termination time they will be run in the order ``C``, ``B``, +``A``. + +**Note:** The functions registered via this module are not called when the +program is killed by a signal not handled by Python, when a Python fatal +internal error is detected, or when :func:`os._exit` is called. .. index:: single: exitfunc (in sys) diff --git a/Misc/NEWS b/Misc/NEWS index 04317f83999..83b5116dc41 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #15233: Python now guarantees that callables registered with the atexit + module will be called in a deterministic order. + - Issue #18747: Re-seed OpenSSL's pseudo-random number generator after fork. A pthread_atfork() child handler is used to seeded the PRNG with pid, time and some stack data. From 7a6b0eecc2cb8b1dca461259485c4ee88859084a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 21 Aug 2013 21:46:12 +0300 Subject: [PATCH 312/480] Issue #17119: Fixed integer overflows when processing large Unicode strings and tuples in the tkinter module. --- Lib/test/test_tcl.py | 17 ++++++++++++++++- Misc/NEWS | 3 +++ Modules/_tkinter.c | 43 +++++++++++++++++++++++++++++++++---------- 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 3da87d9b253..34602623665 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -3,6 +3,7 @@ import unittest import sys import os +import _testcapi from test import test_support from subprocess import Popen, PIPE @@ -245,8 +246,22 @@ def test_split(self): self.assertEqual(split(arg), res) +class BigmemTclTest(unittest.TestCase): + + def setUp(self): + self.interp = Tcl() + + @unittest.skipUnless(_testcapi.INT_MAX < _testcapi.PY_SSIZE_T_MAX, + "needs UINT_MAX < SIZE_MAX") + @test_support.precisionbigmemtest(size=_testcapi.INT_MAX + 1, memuse=5, + dry_run=False) + def test_huge_string(self, size): + value = ' ' * size + self.assertRaises(OverflowError, self.interp.call, 'set', '_', value) + + def test_main(): - test_support.run_unittest(TclTest, TkinterTest) + test_support.run_unittest(TclTest, TkinterTest, BigmemTclTest) if __name__ == "__main__": test_main() diff --git a/Misc/NEWS b/Misc/NEWS index 83b5116dc41..67853e4e9b9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #17119: Fixed integer overflows when processing large Unicode strings + and tuples in the tkinter module. + - Issue #15233: Python now guarantees that callables registered with the atexit module will be called in a deterministic order. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index aed42ac3e4b..a04d580b02e 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -47,6 +47,10 @@ Copyright (C) 1994 Steen Lumholt. #define PyBool_FromLong PyInt_FromLong #endif +#define CHECK_SIZE(size, elemsize) \ + ((size_t)(size) <= (size_t)INT_MAX && \ + (size_t)(size) <= UINT_MAX / (size_t)(elemsize)) + /* Starting with Tcl 8.4, many APIs offer const-correctness. Unfortunately, making _tkinter correct for this API means to break earlier versions. USE_COMPAT_CONST allows to make _tkinter work with both 8.4 and @@ -378,7 +382,7 @@ Merge(PyObject *args) char **argv = NULL; int fvStore[ARGSZ]; int *fv = NULL; - int argc = 0, fvc = 0, i; + Py_ssize_t argc = 0, fvc = 0, i; char *res = NULL; if (!(tmp = PyList_New(0))) @@ -400,8 +404,12 @@ Merge(PyObject *args) argc = PyTuple_Size(args); if (argc > ARGSZ) { - argv = (char **)ckalloc(argc * sizeof(char *)); - fv = (int *)ckalloc(argc * sizeof(int)); + if (!CHECK_SIZE(argc, sizeof(char *))) { + PyErr_SetString(PyExc_OverflowError, "tuple is too long"); + goto finally; + } + argv = (char **)ckalloc((size_t)argc * sizeof(char *)); + fv = (int *)ckalloc((size_t)argc * sizeof(int)); if (argv == NULL || fv == NULL) { PyErr_NoMemory(); goto finally; @@ -983,12 +991,18 @@ AsObj(PyObject *value) else if (PyFloat_Check(value)) return Tcl_NewDoubleObj(PyFloat_AS_DOUBLE(value)); else if (PyTuple_Check(value)) { - Tcl_Obj **argv = (Tcl_Obj**) - ckalloc(PyTuple_Size(value)*sizeof(Tcl_Obj*)); - int i; + Tcl_Obj **argv; + Py_ssize_t size, i; + + size = PyTuple_Size(value); + if (!CHECK_SIZE(size, sizeof(Tcl_Obj *))) { + PyErr_SetString(PyExc_OverflowError, "tuple is too long"); + return NULL; + } + argv = (Tcl_Obj **) ckalloc(((size_t)size) * sizeof(Tcl_Obj *)); if(!argv) return 0; - for(i=0;i= size) outbuf = (Tcl_UniChar*)ckalloc(allocsize); /* Else overflow occurred, and we take the next exit */ @@ -1198,7 +1217,7 @@ static Tcl_Obj** Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) { Tcl_Obj **objv = objStore; - int objc = 0, i; + Py_ssize_t objc = 0, i; if (args == NULL) /* do nothing */; @@ -1213,7 +1232,11 @@ Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) objc = PyTuple_Size(args); if (objc > ARGSZ) { - objv = (Tcl_Obj **)ckalloc(objc * sizeof(char *)); + if (!CHECK_SIZE(objc, sizeof(Tcl_Obj *))) { + PyErr_SetString(PyExc_OverflowError, "tuple is too long"); + return NULL; + } + objv = (Tcl_Obj **)ckalloc(((size_t)objc) * sizeof(Tcl_Obj *)); if (objv == NULL) { PyErr_NoMemory(); objc = 0; From 25bb34ff7f66b322261a48c1fe4a07c7d3e7c3f2 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 22 Aug 2013 00:39:46 +0200 Subject: [PATCH 313/480] Issue #18792: Use "127.0.0.1" or "::1" instead of "localhost" as much as possible, since "localhost" goes through a DNS lookup under recent Windows versions. --- Lib/test/test_asyncore.py | 3 +-- Lib/test/test_ftplib.py | 8 ++++---- Lib/test/test_multiprocessing.py | 4 ++-- Lib/test/test_support.py | 7 ++++++- Misc/NEWS | 4 ++++ 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index 134470fb79a..20eceb63505 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -10,7 +10,7 @@ import struct from test import test_support -from test.test_support import TESTFN, run_unittest, unlink +from test.test_support import TESTFN, run_unittest, unlink, HOST from StringIO import StringIO try: @@ -18,7 +18,6 @@ except ImportError: threading = None -HOST = test_support.HOST class dummysocket: def __init__(self): diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 535bc49f061..e6aaca5376b 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -17,7 +17,7 @@ from unittest import TestCase from test import test_support -from test.test_support import HOST +from test.test_support import HOST, HOSTv6 threading = test_support.import_module('threading') @@ -562,7 +562,7 @@ def test_makepasv(self): class TestIPv6Environment(TestCase): def setUp(self): - self.server = DummyFTPServer((HOST, 0), af=socket.AF_INET6) + self.server = DummyFTPServer((HOSTv6, 0), af=socket.AF_INET6) self.server.start() self.client = ftplib.FTP() self.client.connect(self.server.host, self.server.port) @@ -713,7 +713,7 @@ def testTimeoutDefault(self): self.assertTrue(socket.getdefaulttimeout() is None) socket.setdefaulttimeout(30) try: - ftp = ftplib.FTP("localhost") + ftp = ftplib.FTP(HOST) finally: socket.setdefaulttimeout(None) self.assertEqual(ftp.sock.gettimeout(), 30) @@ -725,7 +725,7 @@ def testTimeoutNone(self): self.assertTrue(socket.getdefaulttimeout() is None) socket.setdefaulttimeout(30) try: - ftp = ftplib.FTP("localhost", timeout=None) + ftp = ftplib.FTP(HOST, timeout=None) finally: socket.setdefaulttimeout(None) self.assertTrue(ftp.sock.gettimeout() is None) diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index 5ffe75994c4..925f39dc4fa 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -1385,7 +1385,7 @@ def test_remote(self): authkey = os.urandom(32) manager = QueueManager( - address=('localhost', 0), authkey=authkey, serializer=SERIALIZER + address=(test.test_support.HOST, 0), authkey=authkey, serializer=SERIALIZER ) manager.start() @@ -1423,7 +1423,7 @@ def _putter(cls, address, authkey): def test_rapid_restart(self): authkey = os.urandom(32) manager = QueueManager( - address=('localhost', 0), authkey=authkey, serializer=SERIALIZER) + address=(test.test_support.HOST, 0), authkey=authkey, serializer=SERIALIZER) srvr = manager.get_server() addr = srvr.address # Close the connection.Listener socket which gets opened as a part diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index fe5a4f35e39..be867bd507e 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -290,7 +290,12 @@ def requires(resource, msg=None): msg = "Use of the `%s' resource not enabled" % resource raise ResourceDenied(msg) -HOST = 'localhost' + +# Don't use "localhost", since resolving it uses the DNS under recent +# Windows versions (see issue #18792). +HOST = "127.0.0.1" +HOSTv6 = "::1" + def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): """Returns an unused port that should be suitable for binding. This is diff --git a/Misc/NEWS b/Misc/NEWS index 67853e4e9b9..318a9744822 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -191,6 +191,10 @@ IDLE Tests ----- +- Issue #18792: Use "127.0.0.1" or "::1" instead of "localhost" as much as + possible, since "localhost" goes through a DNS lookup under recent Windows + versions. + - Issue #18357: add tests for dictview set difference. Patch by Fraser Tweedale. From 2e61f079adf82a98527f29197125c881d40bc94b Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 22 Aug 2013 13:19:48 +0200 Subject: [PATCH 314/480] Issue #18747: Use a parent atfork handler instead of a child atfork handler. fork() is suppose to be async-signal safe but the handler calls unsafe functions. A parent handler mitigates the issue. --- Modules/_ssl.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 1c769edb091..1839dc07114 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1628,23 +1628,25 @@ fails or if it does not provide enough data to seed PRNG."); /* Seed OpenSSL's PRNG at fork(), http://bugs.python.org/issue18747 * - * The child handler seeds the PRNG from pseudo-random data like pid, the - * current time (nanoseconds, miliseconds or seconds) and an uninitialized - * array. The array contains stack variables that are impossible to predict + * The parent handler seeds the PRNG from pseudo-random data like pid, the + * current time (miliseconds or seconds) and an uninitialized arry. + * The array contains stack variables that are impossible to predict * on most systems, e.g. function return address (subject to ASLR), the * stack protection canary and automatic variables. * The code is inspired by Apache's ssl_rand_seed() function. * * Note: * The code uses pthread_atfork() until Python has a proper atfork API. The - * handlers are not removed from the child process. + * handlers are not removed from the child process. A parent handler is used + * instead of a child handler because fork() is suppose to be async-signal + * safe but the handler calls unsafe functions. */ #if defined(HAVE_PTHREAD_ATFORK) && defined(WITH_THREAD) #define PYSSL_RAND_ATFORK 1 static void -PySSL_RAND_atfork_child(void) +PySSL_RAND_atfork_parent(void) { struct { char stack[128]; /* uninitialized (!) stack data, 128 is an @@ -1658,11 +1660,6 @@ PySSL_RAND_atfork_child(void) #endif seed.pid = getpid(); seed.time = time(NULL); - -#if 0 - fprintf(stderr, "PySSL_RAND_atfork_child() seeds %i bytes in pid %i\n", - (int)sizeof(seed), seed.pid); -#endif RAND_add((unsigned char *)&seed, sizeof(seed), 0.0); } @@ -1676,8 +1673,8 @@ PySSL_RAND_atfork(void) return 0; retval = pthread_atfork(NULL, /* prepare */ - NULL, /* parent */ - PySSL_RAND_atfork_child); /* child */ + PySSL_RAND_atfork_parent, /* parent */ + NULL); /* child */ if (retval != 0) { PyErr_SetFromErrno(PyExc_OSError); return -1; From a87671a2bbb60699dc9d0c643f0a53de7f6fb7e4 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 22 Aug 2013 13:22:37 +0200 Subject: [PATCH 315/480] Issue #18747: Update Misc/NEWS to reflect the latest changeset. --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 318a9744822..12ad04739be 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -39,7 +39,7 @@ Library module will be called in a deterministic order. - Issue #18747: Re-seed OpenSSL's pseudo-random number generator after fork. - A pthread_atfork() child handler is used to seeded the PRNG with pid, time + A pthread_atfork() parent handler is used to seeded the PRNG with pid, time and some stack data. - Issue #8865: Concurrent invocation of select.poll.poll() now raises a From cd8bf6e199885c02e866301c50fc669ed1ccce27 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 22 Aug 2013 17:42:45 +0300 Subject: [PATCH 316/480] Issue #16809: Tkinter's splitlist() and split() methods now accept Tcl_Obj argument. This is needed for support Tcl/Tk 8.6. --- Lib/test/test_tcl.py | 4 +++ Misc/NEWS | 3 ++ Modules/_tkinter.c | 65 ++++++++++++++++++++++++++++++++++++-------- 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 34602623665..1e60b1d1a8d 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -209,6 +209,8 @@ def test_splitlist(self): (('a', 3.4), ('a', 3.4)), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), + (call('dict', 'create', 1, u'\u20ac', '\xe2\x82\xac', (3.4,)), + (1, u'\u20ac', u'\u20ac', (3.4,))), ] for arg, res in testcases: self.assertEqual(splitlist(arg), res) @@ -241,6 +243,8 @@ def test_split(self): (('a', (2, 3.4)), ('a', (2, 3.4))), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), + (call('dict', 'create', 12, u'\u20ac', '\xe2\x82\xac', (3.4,)), + (12, u'\u20ac', u'\u20ac', (3.4,))), ] for arg, res in testcases: self.assertEqual(split(arg), res) diff --git a/Misc/NEWS b/Misc/NEWS index 12ad04739be..573961030ef 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #16809: Tkinter's splitlist() and split() methods now accept Tcl_Obj + argument. + - Issue #17119: Fixed integer overflows when processing large Unicode strings and tuples in the tkinter module. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index a04d580b02e..52ee881373f 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -2004,16 +2004,35 @@ Tkapp_SplitList(PyObject *self, PyObject *args) char *list; int argc; char **argv; - PyObject *v; + PyObject *arg, *v; int i; - if (PyTuple_Size(args) == 1) { - v = PyTuple_GetItem(args, 0); - if (PyTuple_Check(v)) { - Py_INCREF(v); - return v; + if (!PyArg_ParseTuple(args, "O:splitlist", &arg)) + return NULL; + if (PyTclObject_Check(arg)) { + int objc; + Tcl_Obj **objv; + if (Tcl_ListObjGetElements(Tkapp_Interp(self), + ((PyTclObject*)arg)->value, + &objc, &objv) == TCL_ERROR) { + return Tkinter_Error(self); + } + if (!(v = PyTuple_New(objc))) + return NULL; + for (i = 0; i < objc; i++) { + PyObject *s = FromObj(self, objv[i]); + if (!s || PyTuple_SetItem(v, i, s)) { + Py_DECREF(v); + return NULL; + } } + return v; } + if (PyTuple_Check(arg)) { + Py_INCREF(arg); + return arg; + } + if (!PyArg_ParseTuple(args, "et:splitlist", "utf-8", &list)) return NULL; @@ -2044,16 +2063,38 @@ Tkapp_SplitList(PyObject *self, PyObject *args) static PyObject * Tkapp_Split(PyObject *self, PyObject *args) { - PyObject *v; + PyObject *arg, *v; char *list; - if (PyTuple_Size(args) == 1) { - PyObject* o = PyTuple_GetItem(args, 0); - if (PyTuple_Check(o)) { - o = SplitObj(o); - return o; + if (!PyArg_ParseTuple(args, "O:split", &arg)) + return NULL; + if (PyTclObject_Check(arg)) { + Tcl_Obj *value = ((PyTclObject*)arg)->value; + int objc; + Tcl_Obj **objv; + int i; + if (Tcl_ListObjGetElements(Tkapp_Interp(self), value, + &objc, &objv) == TCL_ERROR) { + return FromObj(self, value); + } + if (objc == 0) + return PyString_FromString(""); + if (objc == 1) + return FromObj(self, objv[0]); + if (!(v = PyTuple_New(objc))) + return NULL; + for (i = 0; i < objc; i++) { + PyObject *s = FromObj(self, objv[i]); + if (!s || PyTuple_SetItem(v, i, s)) { + Py_DECREF(v); + return NULL; + } } + return v; } + if (PyTuple_Check(arg)) + return SplitObj(arg); + if (!PyArg_ParseTuple(args, "et:split", "utf-8", &list)) return NULL; v = Split(list); From 807dfed1afd2b1c346948a85f15675bf2a74b7fa Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 22 Aug 2013 17:53:16 +0300 Subject: [PATCH 317/480] Issue #16809: Fixed some tkinter incompabilities with Tcl/Tk 8.6. --- Lib/lib-tk/Tkinter.py | 8 ++++---- Lib/lib-tk/test/test_ttk/test_widgets.py | 4 ++-- Misc/NEWS | 2 ++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Lib/lib-tk/Tkinter.py b/Lib/lib-tk/Tkinter.py index 6947bcc41bc..529a3e4b161 100644 --- a/Lib/lib-tk/Tkinter.py +++ b/Lib/lib-tk/Tkinter.py @@ -1347,7 +1347,7 @@ def _grid_configure(self, command, index, cnf, kw): value = words[i+1] if not value: value = None - elif '.' in value: + elif '.' in str(value): value = getdouble(value) else: value = getint(value) @@ -1880,7 +1880,7 @@ def pack_info(self): for i in range(0, len(words), 2): key = words[i][1:] value = words[i+1] - if value[:1] == '.': + if str(value)[:1] == '.': value = self._nametowidget(value) dict[key] = value return dict @@ -1931,7 +1931,7 @@ def place_info(self): for i in range(0, len(words), 2): key = words[i][1:] value = words[i+1] - if value[:1] == '.': + if str(value)[:1] == '.': value = self._nametowidget(value) dict[key] = value return dict @@ -1980,7 +1980,7 @@ def grid_info(self): for i in range(0, len(words), 2): key = words[i][1:] value = words[i+1] - if value[:1] == '.': + if str(value)[:1] == '.': value = self._nametowidget(value) dict[key] = value return dict diff --git a/Lib/lib-tk/test/test_ttk/test_widgets.py b/Lib/lib-tk/test/test_ttk/test_widgets.py index 2fcb3fa7816..c594602dd35 100644 --- a/Lib/lib-tk/test/test_ttk/test_widgets.py +++ b/Lib/lib-tk/test/test_ttk/test_widgets.py @@ -104,7 +104,7 @@ def cb_test(): cbtn['command'] = '' res = cbtn.invoke() - self.assertEqual(res, '') + self.assertEqual(str(res), '') self.assertFalse(len(success) > 1) self.assertEqual(cbtn['offvalue'], cbtn.tk.globalgetvar(cbtn['variable'])) @@ -452,7 +452,7 @@ def cb_test(): cbtn2['command'] = '' res = cbtn2.invoke() - self.assertEqual(res, '') + self.assertEqual(str(res), '') self.assertFalse(len(success) > 1) self.assertEqual(cbtn2['value'], myvar.get()) self.assertEqual(myvar.get(), diff --git a/Misc/NEWS b/Misc/NEWS index 573961030ef..e93858a3339 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,8 @@ Core and Builtins Library ------- +- Issue #16809: Fixed some tkinter incompabilities with Tcl/Tk 8.6. + - Issue #16809: Tkinter's splitlist() and split() methods now accept Tcl_Obj argument. From 891a6a5a319a9d55ce6d37899400adc12dffa073 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Thu, 22 Aug 2013 18:32:53 -0500 Subject: [PATCH 318/480] Add line explaining the "%sort" test. --- Objects/listsort.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/listsort.txt b/Objects/listsort.txt index 31a5445c0e3..44a353a546d 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -60,6 +60,7 @@ Comparison with Python's Samplesort Hybrid /sort: ascending data 3sort: ascending, then 3 random exchanges +sort: ascending, then 10 random at the end + %sort: ascending, then randomly replace 1% of elements w/ random values ~sort: many duplicates =sort: all equal !sort: worst case scenario From 4d0748368434596336ad2a910b2006e2f1f8f8ba Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 23 Aug 2013 13:26:49 -0400 Subject: [PATCH 319/480] - Issue #18709: Fix CVE-2013-4238. The SSL module now handles NULL bytes inside subjectAltName correctly. Formerly the module has used OpenSSL's GENERAL_NAME_print() function to get the string represention of ASN.1 strings for `rfc822Name` (email), `dNSName` (DNS) and `uniformResourceIdentifier` (URI). --- Lib/test/keycert.pem | 59 +++++++++++++------------ Lib/test/nullbytecert.pem | 90 +++++++++++++++++++++++++++++++++++++++ Lib/test/test_ssl.py | 29 ++++++++++++- Misc/NEWS | 9 ++++ Modules/_ssl.c | 64 ++++++++++++++++++++++++++-- 5 files changed, 216 insertions(+), 35 deletions(-) create mode 100644 Lib/test/nullbytecert.pem diff --git a/Lib/test/keycert.pem b/Lib/test/keycert.pem index 2f46fcf1c62..64318aa2e03 100644 --- a/Lib/test/keycert.pem +++ b/Lib/test/keycert.pem @@ -1,32 +1,31 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L -opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH -fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB -AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU -D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA -IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM -oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 -ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ -loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j -oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA -z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq -ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV -q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD -VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x -IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT -U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 -NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl -bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj -aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh -m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 -M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn -fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC -AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb -08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx -CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ -iHkC6gGdBJhogs4= +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX -----END CERTIFICATE----- diff --git a/Lib/test/nullbytecert.pem b/Lib/test/nullbytecert.pem new file mode 100644 index 00000000000..447186c9508 --- /dev/null +++ b/Lib/test/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index c1e53ec1a67..3b868f04088 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -31,6 +31,7 @@ HOST = test_support.HOST CERTFILE = None SVN_PYTHON_ORG_ROOT_CERT = None +NULLBYTECERT = None def handle_error(prefix): exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) @@ -88,6 +89,27 @@ def test_parse_cert(self): if test_support.verbose: sys.stdout.write("\n" + pprint.pformat(p) + "\n") + def test_parse_cert_CVE_2013_4073(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if test_support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + self.assertEqual(p['subjectAltName'], + (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + ) + def test_DER_to_PEM(self): with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f: pem = f.read() @@ -1210,15 +1232,18 @@ def test_main(verbose=False): if skip_expected: raise test_support.TestSkipped("No SSL support") - global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT + global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, NULLBYTECERT CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "keycert.pem") SVN_PYTHON_ORG_ROOT_CERT = os.path.join( os.path.dirname(__file__) or os.curdir, "https_svn_python_org_root.pem") + NULLBYTECERT = os.path.join(os.path.dirname(__file__) or os.curdir, + "nullbytecert.pem") if (not os.path.exists(CERTFILE) or - not os.path.exists(SVN_PYTHON_ORG_ROOT_CERT)): + not os.path.exists(SVN_PYTHON_ORG_ROOT_CERT) or + not os.path.exists(NULLBYTECERT)): raise test_support.TestFailed("Can't read certificate files!") tests = [BasicTests] diff --git a/Misc/NEWS b/Misc/NEWS index 2a865df43f2..790dc583b36 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -16,6 +16,15 @@ Library - Issue #16248: Disable code execution from the user's home directory by tkinter when the -E flag is passed to Python. Patch by Zachary Ware. +Extension Modules +----------------- + +- Issue #18709: Fix CVE-2013-4238. The SSL module now handles NULL bytes + inside subjectAltName correctly. Formerly the module has used OpenSSL's + GENERAL_NAME_print() function to get the string represention of ASN.1 + strings for `rfc822Name` (email), `dNSName` (DNS) and + `uniformResourceIdentifier` (URI). + What's New in Python 2.6.8? =========================== diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 46f71e858e2..a045be421c4 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -302,8 +302,10 @@ newPySSLObject(PySocketSockObject *Sock, char *key_file, char *cert_file, self->ctx = SSL_CTX_new(TLSv1_method()); /* Set up context */ else if (proto_version == PY_SSL_VERSION_SSL3) self->ctx = SSL_CTX_new(SSLv3_method()); /* Set up context */ +#ifndef OPENSSL_NO_SSL2 else if (proto_version == PY_SSL_VERSION_SSL2) self->ctx = SSL_CTX_new(SSLv2_method()); /* Set up context */ +#endif else if (proto_version == PY_SSL_VERSION_SSL23) self->ctx = SSL_CTX_new(SSLv23_method()); /* Set up context */ PySSL_END_ALLOW_THREADS @@ -718,8 +720,13 @@ _get_peer_alt_names (X509 *certificate) { /* get a rendering of each name in the set of names */ + int gntype; + ASN1_STRING *as = NULL; + name = sk_GENERAL_NAME_value(names, j); - if (name->type == GEN_DIRNAME) { + gntype = name-> type; + switch (gntype) { + case GEN_DIRNAME: /* we special-case DirName as a tuple of tuples of attributes */ @@ -741,11 +748,61 @@ _get_peer_alt_names (X509 *certificate) { goto fail; } PyTuple_SET_ITEM(t, 1, v); + break; - } else { + case GEN_EMAIL: + case GEN_DNS: + case GEN_URI: + /* GENERAL_NAME_print() doesn't handle NUL bytes in ASN1_string + correctly. */ + t = PyTuple_New(2); + if (t == NULL) + goto fail; + switch (gntype) { + case GEN_EMAIL: + v = PyUnicode_FromString("email"); + as = name->d.rfc822Name; + break; + case GEN_DNS: + v = PyUnicode_FromString("DNS"); + as = name->d.dNSName; + break; + case GEN_URI: + v = PyUnicode_FromString("URI"); + as = name->d.uniformResourceIdentifier; + break; + } + if (v == NULL) { + Py_DECREF(t); + goto fail; + } + PyTuple_SET_ITEM(t, 0, v); + v = PyString_FromStringAndSize((char *)ASN1_STRING_data(as), + ASN1_STRING_length(as)); + if (v == NULL) { + Py_DECREF(t); + goto fail; + } + PyTuple_SET_ITEM(t, 1, v); + break; + default: /* for everything else, we use the OpenSSL print form */ - + switch (gntype) { + /* check for new general name type */ + case GEN_OTHERNAME: + case GEN_X400: + case GEN_EDIPARTY: + case GEN_IPADD: + case GEN_RID: + break; + default: + if (PyErr_Warn(PyExc_RuntimeWarning, + "Unknown general name type") == -1) { + goto fail; + } + break; + } (void) BIO_reset(biobuf); GENERAL_NAME_print(biobuf, name); len = BIO_gets(biobuf, buf, sizeof(buf)-1); @@ -771,6 +828,7 @@ _get_peer_alt_names (X509 *certificate) { goto fail; } PyTuple_SET_ITEM(t, 1, v); + break; } /* and add that rendering to the list */ From 309de7d196d404919f5a6e81f536e1a23250f757 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Fri, 23 Aug 2013 23:06:31 +0300 Subject: [PATCH 320/480] #18798: fix typo in test_fcntl. Patch by Vajrasky Kok. --- Lib/test/test_fcntl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 7a35f61396d..d7e409cefc6 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -11,7 +11,7 @@ from test.test_support import (verbose, TESTFN, unlink, run_unittest, import_module) -# Skip test if no fnctl module. +# Skip test if no fcntl module. fcntl = import_module('fcntl') From b986b607d045ada2639ae5d3c5d04cd2709b2aad Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 24 Aug 2013 00:28:38 +0300 Subject: [PATCH 321/480] Issue #18757: Improved cross-references in the concurrent package. --- Doc/library/multiprocessing.rst | 63 +++++++++++++++++++-------------- Doc/library/queue.rst | 6 ++-- Doc/library/select.rst | 16 ++++----- Doc/library/subprocess.rst | 18 +++++----- Doc/library/threading.rst | 13 +++---- 5 files changed, 65 insertions(+), 51 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 6e8c3a586e7..86e8d24c818 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -382,7 +382,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the Unix daemons or services, they are normal processes that will be terminated (and not joined) if non-daemonic processes have exited. - In addition to the :class:`Threading.Thread` API, :class:`Process` objects + In addition to the :class:`threading.Thread` API, :class:`Process` objects also support the following attributes and methods: .. attribute:: pid @@ -401,7 +401,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the The process's authentication key (a byte string). When :mod:`multiprocessing` is initialized the main process is assigned a - random string using :func:`os.random`. + random string using :func:`os.urandom`. When a :class:`Process` object is created, it will inherit the authentication key of its parent process, although this may be changed by @@ -518,7 +518,8 @@ Note that one can also create a shared queue by using a manager object -- see .. warning:: As mentioned above, if a child process has put items on a queue (and it has - not used :meth:`JoinableQueue.cancel_join_thread`), then that process will + not used :meth:`JoinableQueue.cancel_join_thread + `), then that process will not terminate until all buffered items have been flushed to the pipe. This means that if you try joining that process you may get a deadlock unless @@ -667,7 +668,7 @@ For an example of the usage of queues for interprocess communication see call to :meth:`task_done` tells the queue that the processing on the task is complete. - If a :meth:`~Queue.join` is currently blocking, it will resume when all + If a :meth:`~Queue.Queue.join` is currently blocking, it will resume when all items have been processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`~Queue.put` into the queue). @@ -683,7 +684,7 @@ For an example of the usage of queues for interprocess communication see queue. The count goes down whenever a consumer thread calls :meth:`task_done` to indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, - :meth:`~Queue.join` unblocks. + :meth:`~Queue.Queue.join` unblocks. Miscellaneous @@ -1049,8 +1050,9 @@ processes. array. If *lock* is ``True`` (the default) then a new lock object is created to - synchronize access to the value. If *lock* is a :class:`Lock` or - :class:`RLock` object then that will be used to synchronize access to the + synchronize access to the value. If *lock* is a + :class:`~multiprocessing.Lock` or :class:`~multiprocessing.RLock` object + then that will be used to synchronize access to the value. If *lock* is ``False`` then access to the returned object will not be automatically protected by a lock, so it will not necessarily be "process-safe". @@ -1064,8 +1066,8 @@ processes. object. If *lock* is ``True`` (the default) then a new lock object is created to - synchronize access to the value. If *lock* is a :class:`Lock` or - :class:`RLock` object then that will be used to synchronize access to the + synchronize access to the value. If *lock* is a :class:`~multiprocessing.Lock` or + :class:`~multiprocessing.RLock` object then that will be used to synchronize access to the value. If *lock* is ``False`` then access to the returned object will not be automatically protected by a lock, so it will not necessarily be "process-safe". @@ -1247,8 +1249,8 @@ their parent process exits. The manager classes are defined in the :attr:`proxytype._exposed_` is used instead if it exists.) In the case where no exposed list is specified, all "public methods" of the shared object will be accessible. (Here a "public method" means any attribute - which has a :meth:`__call__` method and whose name does not begin with - ``'_'``.) + which has a :meth:`~object.__call__` method and whose name does not begin + with ``'_'``.) *method_to_typeid* is a mapping used to specify the return type of those exposed methods which should return a proxy. It maps method names to @@ -1758,7 +1760,8 @@ Listeners and Clients :synopsis: API for dealing with sockets. Usually message passing between processes is done using queues or by using -:class:`Connection` objects returned by :func:`Pipe`. +:class:`~multiprocessing.Connection` objects returned by +:func:`~multiprocessing.Pipe`. However, the :mod:`multiprocessing.connection` module allows some extra flexibility. It basically gives a high level message oriented API for dealing @@ -1824,7 +1827,8 @@ authentication* using the :mod:`hmac` module. private temporary directory created using :func:`tempfile.mkstemp`. If the listener object uses a socket then *backlog* (1 by default) is passed - to the :meth:`listen` method of the socket once it has been bound. + to the :meth:`~socket.socket.listen` method of the socket once it has been + bound. If *authenticate* is ``True`` (``False`` by default) or *authkey* is not ``None`` then digest authentication is used. @@ -1841,8 +1845,9 @@ authentication* using the :mod:`hmac` module. .. method:: accept() Accept a connection on the bound socket or named pipe of the listener - object and return a :class:`Connection` object. If authentication is - attempted and fails, then :exc:`AuthenticationError` is raised. + object and return a :class:`~multiprocessing.Connection` object. If + authentication is attempted and fails, then + :exc:`~multiprocessing.AuthenticationError` is raised. .. method:: close() @@ -1938,7 +1943,8 @@ an ``'AF_PIPE'`` address rather than an ``'AF_UNIX'`` address. Authentication keys ~~~~~~~~~~~~~~~~~~~ -When one uses :meth:`Connection.recv`, the data received is automatically +When one uses :meth:`Connection.recv `, the +data received is automatically unpickled. Unfortunately unpickling data from an untrusted source is a security risk. Therefore :class:`Listener` and :func:`Client` use the :mod:`hmac` module to provide digest authentication. @@ -2087,9 +2093,10 @@ Joining zombie processes On Unix when a process finishes but has not been joined it becomes a zombie. There should never be very many because each time a new process starts (or - :func:`active_children` is called) all completed processes which have not - yet been joined will be joined. Also calling a finished process's - :meth:`Process.is_alive` will join the process. Even so it is probably good + :func:`~multiprocessing.active_children` is called) all completed processes + which have not yet been joined will be joined. Also calling a finished + process's :meth:`Process.is_alive ` will + join the process. Even so it is probably good practice to explicitly join all the processes that you start. Better to inherit than pickle/unpickle @@ -2102,13 +2109,15 @@ Better to inherit than pickle/unpickle Avoid terminating processes - Using the :meth:`Process.terminate` method to stop a process is liable to + Using the :meth:`Process.terminate ` + method to stop a process is liable to cause any shared resources (such as locks, semaphores, pipes and queues) currently being used by the process to become broken or unavailable to other processes. Therefore it is probably best to only consider using - :meth:`Process.terminate` on processes which never use any shared resources. + :meth:`Process.terminate ` on processes + which never use any shared resources. Joining processes that use queues @@ -2192,7 +2201,7 @@ Beware of replacing :data:`sys.stdin` with a "file like object" resulting in a bad file descriptor error, but introduces a potential danger to applications which replace :func:`sys.stdin` with a "file-like object" with output buffering. This danger is that if multiple processes call - :func:`close()` on this file-like object, it could result in the same + :meth:`~io.IOBase.close()` on this file-like object, it could result in the same data being flushed to the object multiple times, resulting in corruption. If you write a file-like object and implement your own caching, you can @@ -2221,14 +2230,16 @@ More picklability as the ``target`` argument on Windows --- just define a function and use that instead. - Also, if you subclass :class:`Process` then make sure that instances will be - picklable when the :meth:`Process.start` method is called. + Also, if you subclass :class:`~multiprocessing.Process` then make sure that + instances will be picklable when the :meth:`Process.start + ` method is called. Global variables Bear in mind that if code run in a child process tries to access a global variable, then the value it sees (if any) may not be the same as the value - in the parent process at the time that :meth:`Process.start` was called. + in the parent process at the time that :meth:`Process.start + ` was called. However, global variables which are just module level constants cause no problems. @@ -2283,7 +2294,7 @@ Demonstration of how to create and use customized managers and proxies: .. literalinclude:: ../includes/mp_newtype.py -Using :class:`Pool`: +Using :class:`~multiprocessing.pool.Pool`: .. literalinclude:: ../includes/mp_pool.py diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 179c4c67c69..b52570550da 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -60,13 +60,15 @@ The :mod:`Queue` module defines the following classes and exceptions: .. exception:: Empty - Exception raised when non-blocking :meth:`get` (or :meth:`get_nowait`) is called + Exception raised when non-blocking :meth:`~Queue.get` (or + :meth:`~Queue.get_nowait`) is called on a :class:`Queue` object which is empty. .. exception:: Full - Exception raised when non-blocking :meth:`put` (or :meth:`put_nowait`) is called + Exception raised when non-blocking :meth:`~Queue.put` (or + :meth:`~Queue.put_nowait`) is called on a :class:`Queue` object which is full. .. seealso:: diff --git a/Doc/library/select.rst b/Doc/library/select.rst index d131cb98fba..24cb7567c88 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -63,7 +63,7 @@ The module defines the following: This is a straightforward interface to the Unix :c:func:`select` system call. The first three arguments are sequences of 'waitable objects': either integers representing file descriptors or objects with a parameterless method - named :meth:`fileno` returning such an integer: + named :meth:`~io.IOBase.fileno` returning such an integer: * *rlist*: wait until ready for reading * *wlist*: wait until ready for writing @@ -88,8 +88,8 @@ The module defines the following: Among the acceptable object types in the sequences are Python file objects (e.g. ``sys.stdin``, or objects returned by :func:`open` or :func:`os.popen`), socket objects returned by :func:`socket.socket`. You may also define a :dfn:`wrapper` - class yourself, as long as it has an appropriate :meth:`fileno` method (that - really returns a file descriptor, not just a random integer). + class yourself, as long as it has an appropriate :meth:`~io.IOBase.fileno` + method (that really returns a file descriptor, not just a random integer). .. note:: @@ -207,10 +207,10 @@ linearly scanned again. :c:func:`select` is O(highest file descriptor), while .. method:: poll.register(fd[, eventmask]) Register a file descriptor with the polling object. Future calls to the - :meth:`poll` method will then check whether the file descriptor has any pending - I/O events. *fd* can be either an integer, or an object with a :meth:`fileno` - method that returns an integer. File objects implement :meth:`fileno`, so they - can also be used as the argument. + :meth:`poll` method will then check whether the file descriptor has any + pending I/O events. *fd* can be either an integer, or an object with a + :meth:`~io.IOBase.fileno` method that returns an integer. File objects + implement :meth:`!fileno`, so they can also be used as the argument. *eventmask* is an optional bitmask describing the type of events you want to check for, and can be a combination of the constants :const:`POLLIN`, @@ -251,7 +251,7 @@ linearly scanned again. :c:func:`select` is O(highest file descriptor), while Remove a file descriptor being tracked by a polling object. Just like the :meth:`register` method, *fd* can be an integer or an object with a - :meth:`fileno` method that returns an integer. + :meth:`~io.IOBase.fileno` method that returns an integer. Attempting to remove a file descriptor that was never registered causes a :exc:`KeyError` exception to be raised. diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index d29fbecfa12..c92a43fd3c1 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -74,7 +74,7 @@ use cases, the underlying :class:`Popen` interface can be used directly. Run command with arguments. Wait for command to complete. If the return code was zero then return, otherwise raise :exc:`CalledProcessError`. The :exc:`CalledProcessError` object will have the return code in the - :attr:`returncode` attribute. + :attr:`~CalledProcessError.returncode` attribute. The arguments shown above are merely the most common ones, described below in :ref:`frequently-used-arguments` (hence the slightly odd notation in @@ -114,8 +114,8 @@ use cases, the underlying :class:`Popen` interface can be used directly. If the return code was non-zero it raises a :exc:`CalledProcessError`. The :exc:`CalledProcessError` object will have the return code in the - :attr:`returncode` attribute and any output in the :attr:`output` - attribute. + :attr:`~CalledProcessError.returncode` attribute and any output in the + :attr:`~CalledProcessError.output` attribute. The arguments shown above are merely the most common ones, described below in :ref:`frequently-used-arguments` (hence the slightly odd notation in @@ -462,14 +462,14 @@ Instances of the :class:`Popen` class have the following methods: .. method:: Popen.poll() - Check if child process has terminated. Set and return :attr:`returncode` - attribute. + Check if child process has terminated. Set and return + :attr:`~Popen.returncode` attribute. .. method:: Popen.wait() - Wait for child process to terminate. Set and return :attr:`returncode` - attribute. + Wait for child process to terminate. Set and return + :attr:`~Popen.returncode` attribute. .. warning:: @@ -695,8 +695,8 @@ In this section, "a becomes b" means that b can be used as a replacement for a. In addition, the replacements using :func:`check_output` will fail with a :exc:`CalledProcessError` if the requested operation produces a non-zero - return code. The output is still available as the ``output`` attribute of - the raised exception. + return code. The output is still available as the + :attr:`~CalledProcessError.output` attribute of the raised exception. In the following examples, we assume that the relevant functions have already been imported from the :mod:`subprocess` module. diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 1f4b5a6fee5..9e56dc0ed89 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -167,7 +167,7 @@ This module defines the following functions and objects: Set a trace function for all threads started from the :mod:`threading` module. The *func* will be passed to :func:`sys.settrace` for each thread, before its - :meth:`run` method is called. + :meth:`~Thread.run` method is called. .. versionadded:: 2.3 @@ -178,7 +178,7 @@ This module defines the following functions and objects: Set a profile function for all threads started from the :mod:`threading` module. The *func* will be passed to :func:`sys.setprofile` for each thread, before its - :meth:`run` method is called. + :meth:`~Thread.run` method is called. .. versionadded:: 2.3 @@ -742,10 +742,11 @@ This class represents an action that should be run only after a certain amount of time has passed --- a timer. :class:`Timer` is a subclass of :class:`Thread` and as such also functions as an example of creating custom threads. -Timers are started, as with threads, by calling their :meth:`start` method. The -timer can be stopped (before its action has begun) by calling the :meth:`cancel` -method. The interval the timer will wait before executing its action may not be -exactly the same as the interval specified by the user. +Timers are started, as with threads, by calling their :meth:`~Timer.start` +method. The timer can be stopped (before its action has begun) by calling the +:meth:`~Timer.cancel` method. The interval the timer will wait before +executing its action may not be exactly the same as the interval specified by +the user. For example:: From ed168775f38b6d65548a257d39fcf9f889e94e9d Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 24 Aug 2013 20:52:27 +0200 Subject: [PATCH 322/480] Issue #18756: make test_urandom_failure more robust by executing its code in a subprocess --- Lib/test/test_os.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index bc7dfbea056..1d7e836f5c9 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -16,6 +16,7 @@ resource = None from test import test_support +from test.script_helper import assert_python_ok import mmap import uuid @@ -569,17 +570,26 @@ def test_urandom_subprocess(self): @unittest.skipUnless(resource, "test requires the resource module") def test_urandom_failure(self): - soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) - resource.setrlimit(resource.RLIMIT_NOFILE, (1, hard_limit)) - try: - with self.assertRaises(OSError) as cm: + # Check urandom() failing when it is not able to open /dev/random. + # We spawn a new process to make the test more robust (if getrlimit() + # failed to restore the file descriptor limit after this, the whole + # test suite would crash; this actually happened on the OS X Tiger + # buildbot). + code = """if 1: + import errno + import os + import resource + + soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + resource.setrlimit(resource.RLIMIT_NOFILE, (1, hard_limit)) + try: os.urandom(16) - self.assertEqual(cm.exception.errno, errno.EMFILE) - finally: - # We restore the old limit as soon as possible. If doing it - # using addCleanup(), code running in between would fail - # creating any file descriptor. - resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) + except OSError as e: + assert e.errno == errno.EMFILE, e.errno + else: + raise AssertionError("OSError not raised") + """ + assert_python_ok('-c', code) class ExecvpeTests(unittest.TestCase): From bd7feb716ad3d6d44ed7e4d8d71938ff312c0d35 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sat, 24 Aug 2013 15:15:19 -0500 Subject: [PATCH 323/480] Various clarifications based on feedback & questions over the years. --- Objects/listsort.txt | 115 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 96 insertions(+), 19 deletions(-) diff --git a/Objects/listsort.txt b/Objects/listsort.txt index 44a353a546d..08fef96bdf0 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -100,11 +100,13 @@ Comparison with Python's Samplesort Hybrid The algorithms are effectively identical in these cases, except that timsort does one less compare in \sort. - Now for the more interesting cases. lg(n!) is the information-theoretic - limit for the best any comparison-based sorting algorithm can do on - average (across all permutations). When a method gets significantly - below that, it's either astronomically lucky, or is finding exploitable - structure in the data. + Now for the more interesting cases. Where lg(x) is the logarithm of x to + the base 2 (e.g., lg(8)=3), lg(n!) is the information-theoretic limit for + the best any comparison-based sorting algorithm can do on average (across + all permutations). When a method gets significantly below that, it's + either astronomically lucky, or is finding exploitable structure in the + data. + n lg(n!) *sort 3sort +sort %sort ~sort !sort ------- ------- ------ ------- ------- ------ ------- -------- @@ -251,7 +253,7 @@ Computing minrun ---------------- If N < 64, minrun is N. IOW, binary insertion sort is used for the whole array then; it's hard to beat that given the overheads of trying something -fancier. +fancier (see note BINSORT). When N is a power of 2, testing on random data showed that minrun values of 16, 32, 64 and 128 worked about equally well. At 256 the data-movement cost @@ -379,10 +381,10 @@ with wildly unbalanced run lengths. Merge Memory ------------ -Merging adjacent runs of lengths A and B in-place is very difficult. -Theoretical constructions are known that can do it, but they're too difficult -and slow for practical use. But if we have temp memory equal to min(A, B), -it's easy. +Merging adjacent runs of lengths A and B in-place, and in linear time, is +difficult. Theoretical constructions are known that can do it, but they're +too difficult and slow for practical use. But if we have temp memory equal +to min(A, B), it's easy. If A is smaller (function merge_lo), copy A to a temp array, leave B alone, and then we can do the obvious merge algorithm left to right, from the temp @@ -457,10 +459,10 @@ finding the right spot early in B (more on that later). After finding such a k, the region of uncertainty is reduced to 2**(k-1) - 1 consecutive elements, and a straight binary search requires exactly k-1 -additional comparisons to nail it. Then we copy all the B's up to that -point in one chunk, and then copy A[0]. Note that no matter where A[0] -belongs in B, the combination of galloping + binary search finds it in no -more than about 2*lg(B) comparisons. +additional comparisons to nail it (see note REGION OF UNCERTAINTY). Then we +copy all the B's up to that point in one chunk, and then copy A[0]. Note +that no matter where A[0] belongs in B, the combination of galloping + binary +search finds it in no more than about 2*lg(B) comparisons. If we did a straight binary search, we could find it in no more than ceiling(lg(B+1)) comparisons -- but straight binary search takes that many @@ -573,11 +575,11 @@ Galloping Complication The description above was for merge_lo. merge_hi has to merge "from the other end", and really needs to gallop starting at the last element in a run instead of the first. Galloping from the first still works, but does more -comparisons than it should (this is significant -- I timed it both ways). -For this reason, the gallop_left() and gallop_right() functions have a -"hint" argument, which is the index at which galloping should begin. So -galloping can actually start at any index, and proceed at offsets of 1, 3, -7, 15, ... or -1, -3, -7, -15, ... from the starting index. +comparisons than it should (this is significant -- I timed it both ways). For +this reason, the gallop_left() and gallop_right() (see note LEFT OR RIGHT) +functions have a "hint" argument, which is the index at which galloping +should begin. So galloping can actually start at any index, and proceed at +offsets of 1, 3, 7, 15, ... or -1, -3, -7, -15, ... from the starting index. In the code as I type it's always called with either 0 or n-1 (where n is the # of elements in a run). It's tempting to try to do something fancier, @@ -676,3 +678,78 @@ immediately. The consequence is that it ends up using two compares to sort [2, 1]. Gratifyingly, timsort doesn't do any special-casing, so had to be taught how to deal with mixtures of ascending and descending runs efficiently in all cases. + + +NOTES +----- + +BINSORT +A "binary insertion sort" is just like a textbook insertion sort, but instead +of locating the correct position of the next item via linear (one at a time) +search, an equivalent to Python's bisect.bisect_right is used to find the +correct position in logarithmic time. Most texts don't mention this +variation, and those that do usually say it's not worth the bother: insertion +sort remains quadratic (expected and worst cases) either way. Speeding the +search doesn't reduce the quadratic data movement costs. + +But in CPython's case, comparisons are extraordinarily expensive compared to +moving data, and the details matter. Moving objects is just copying +pointers. Comparisons can be arbitrarily expensive (can invoke arbitary +user-supplied Python code), but even in simple cases (like 3 < 4) _all_ +decisions are made at runtime: what's the type of the left comparand? the +type of the right? do they need to be coerced to a common type? where's the +code to compare these types? And so on. Even the simplest Python comparison +triggers a large pile of C-level pointer dereferences, conditionals, and +function calls. + +So cutting the number of compares is almost always measurably helpful in +CPython, and the savings swamp the quadratic-time data movement costs for +reasonable minrun values. + + +LEFT OR RIGHT +gallop_left() and gallop_right() are akin to the Python bisect module's +bisect_left() and bisect_right(): they're the same unless the slice they're +searching contains a (at least one) value equal to the value being searched +for. In that case, gallop_left() returns the position immediately before the +leftmost equal value, and gallop_right() the position immediately after the +rightmost equal value. The distinction is needed to preserve stability. In +general, when merging adjacent runs A and B, gallop_left is used to search +thru B for where an element from A belongs, and gallop_right to search thru A +for where an element from B belongs. + + +REGION OF UNCERTAINTY +Two kinds of confusion seem to be common about the claim that after finding +a k such that + + B[2**(k-1) - 1] < A[0] <= B[2**k - 1] + +then a binary search requires exactly k-1 tries to find A[0]'s proper +location. For concreteness, say k=3, so B[3] < A[0] <= B[7]. + +The first confusion takes the form "OK, then the region of uncertainty is at +indices 3, 4, 5, 6 and 7: that's 5 elements, not the claimed 2**(k-1) - 1 = +3"; or the region is viewed as a Python slice and the objection is "but that's +the slice B[3:7], so has 7-3 = 4 elements". Resolution: we've already +compared A[0] against B[3] and against B[7], so A[0]'s correct location is +already known wrt _both_ endpoints. What remains is to find A[0]'s correct +location wrt B[4], B[5] and B[6], which spans 3 elements. Or in general, the +slice (leaving off both endpoints) (2**(k-1)-1)+1 through (2**k-1)-1 +inclusive = 2**(k-1) through (2**k-1)-1 inclusive, which has + (2**k-1)-1 - 2**(k-1) + 1 = + 2**k-1 - 2**(k-1) = + 2*2**k-1 - 2**(k-1) = + (2-1)*2**(k-1) - 1 = + 2**(k-1) - 1 +elements. + +The second confusion: "k-1 = 2 binary searches can find the correct location +among 2**(k-1) = 4 elements, but you're only applying it to 3 elements: we +could make this more efficient by arranging for the region of uncertainty to +span 2**(k-1) elements." Resolution: that confuses "elements" with +"locations". In a slice with N elements, there are N+1 _locations_. In the +example, with the region of uncertainty B[4], B[5], B[6], there are 4 +locations: before B[4], between B[4] and B[5], between B[5] and B[6], and +after B[6]. In general, across 2**(k-1)-1 elements, there are 2**(k-1) +locations. That's why k-1 binary searches are necessary and sufficient. From 1c857e316b5fbc2a2bf37cfdf1e0944912449ccb Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 25 Aug 2013 14:12:41 +0200 Subject: [PATCH 324/480] Issue #18709: Fix issue with IPv6 address in subjectAltName on Mac OS X Tiger --- Lib/test/test_ssl.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 563f9293fef..f5390f68067 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -137,13 +137,21 @@ def test_parse_cert_CVE_2013_4238(self): (('emailAddress', 'python-dev@python.org'),)) self.assertEqual(p['subject'], subject) self.assertEqual(p['issuer'], subject) - self.assertEqual(p['subjectAltName'], - (('DNS', 'altnull.python.org\x00example.com'), - ('email', 'null@python.org\x00user@example.org'), - ('URI', 'http://null.python.org\x00http://example.org'), - ('IP Address', '192.0.2.1'), - ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) - ) + if ssl._OPENSSL_API_VERSION >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '')) + + self.assertEqual(p['subjectAltName'], san) def test_DER_to_PEM(self): with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f: From c931e3b5641bf8f342e98073897a7c842dd8f698 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 25 Aug 2013 14:12:41 +0200 Subject: [PATCH 325/480] Issue #18709: Fix issue with IPv6 address in subjectAltName on Mac OS X Tiger --- Lib/test/test_ssl.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 3b868f04088..f779dbae226 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -102,13 +102,21 @@ def test_parse_cert_CVE_2013_4073(self): (('emailAddress', 'python-dev@python.org'),)) self.assertEqual(p['subject'], subject) self.assertEqual(p['issuer'], subject) - self.assertEqual(p['subjectAltName'], - (('DNS', 'altnull.python.org\x00example.com'), - ('email', 'null@python.org\x00user@example.org'), - ('URI', 'http://null.python.org\x00http://example.org'), - ('IP Address', '192.0.2.1'), - ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) - ) + if ssl._OPENSSL_API_VERSION >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '')) + + self.assertEqual(p['subjectAltName'], san) def test_DER_to_PEM(self): with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f: From c19138e397e7ba4e94f981895ee87f5da2931ee5 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 25 Aug 2013 14:19:16 +0200 Subject: [PATCH 326/480] Issue #18747: Fix spelling errors in my commit message and comments, thanks to Vajrasky Kok for proof-reading. --- Misc/NEWS | 2 +- Modules/_ssl.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index e93858a3339..3e1a4861206 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -44,7 +44,7 @@ Library module will be called in a deterministic order. - Issue #18747: Re-seed OpenSSL's pseudo-random number generator after fork. - A pthread_atfork() parent handler is used to seeded the PRNG with pid, time + A pthread_atfork() parent handler is used to seed the PRNG with pid, time and some stack data. - Issue #8865: Concurrent invocation of select.poll.poll() now raises a diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 1839dc07114..5ddbb1dc222 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1629,7 +1629,7 @@ fails or if it does not provide enough data to seed PRNG."); /* Seed OpenSSL's PRNG at fork(), http://bugs.python.org/issue18747 * * The parent handler seeds the PRNG from pseudo-random data like pid, the - * current time (miliseconds or seconds) and an uninitialized arry. + * current time (miliseconds or seconds) and an uninitialized array. * The array contains stack variables that are impossible to predict * on most systems, e.g. function return address (subject to ASLR), the * stack protection canary and automatic variables. @@ -1638,7 +1638,7 @@ fails or if it does not provide enough data to seed PRNG."); * Note: * The code uses pthread_atfork() until Python has a proper atfork API. The * handlers are not removed from the child process. A parent handler is used - * instead of a child handler because fork() is suppose to be async-signal + * instead of a child handler because fork() is supposed to be async-signal * safe but the handler calls unsafe functions. */ From b4df68634a64bdfccc15f093417e00c5bdcce0e6 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 25 Aug 2013 14:44:27 +0200 Subject: [PATCH 327/480] Fix test --- Lib/test/test_ssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index f5390f68067..9c33c8f421a 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -137,7 +137,7 @@ def test_parse_cert_CVE_2013_4238(self): (('emailAddress', 'python-dev@python.org'),)) self.assertEqual(p['subject'], subject) self.assertEqual(p['issuer'], subject) - if ssl._OPENSSL_API_VERSION >= (0, 9, 8): + if ssl.OPENSSL_VERSION_INFO >= (0, 9, 8): san = (('DNS', 'altnull.python.org\x00example.com'), ('email', 'null@python.org\x00user@example.org'), ('URI', 'http://null.python.org\x00http://example.org'), From 22815b545f7bcb46d4ced6d4825662489800f58a Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 25 Aug 2013 14:57:00 +0200 Subject: [PATCH 328/480] Issue #11973: Fix a problem in kevent. The flags and fflags fields are now properly handled as unsigned. --- Misc/NEWS | 3 +++ Modules/selectmodule.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 3e1a4861206..c295fc39aca 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #11973: Fix a problem in kevent. The flags and fflags fields are now + properly handled as unsigned. + - Issue #16809: Fixed some tkinter incompabilities with Tcl/Tk 8.6. - Issue #16809: Tkinter's splitlist() and split() methods now accept Tcl_Obj diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 00d5f6c058c..6a3652160a9 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -1270,7 +1270,7 @@ kqueue_event_init(kqueue_event_Object *self, PyObject *args, PyObject *kwds) PyObject *pfd; static char *kwlist[] = {"ident", "filter", "flags", "fflags", "data", "udata", NULL}; - static char *fmt = "O|hhi" DATA_FMT_UNIT UINTPTRT_FMT_UNIT ":kevent"; + static char *fmt = "O|hHI" DATA_FMT_UNIT UINTPTRT_FMT_UNIT ":kevent"; EV_SET(&(self->e), 0, EVFILT_READ, EV_ADD, 0, 0, 0); /* defaults */ From 4831433e9e478c93b31950c8569196b057949b41 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 25 Aug 2013 19:12:33 +0300 Subject: [PATCH 329/480] Issue #18817: Fix a resource warning in Lib/aifc.py demo. --- Lib/aifc.py | 42 +++++++++++++++++++++++------------------- Misc/NEWS | 2 ++ 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/Lib/aifc.py b/Lib/aifc.py index c2aad1ba66b..12c665fa8c1 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -953,23 +953,27 @@ def open(f, mode=None): sys.argv.append('/usr/demos/data/audio/bach.aiff') fn = sys.argv[1] f = open(fn, 'r') - print "Reading", fn - print "nchannels =", f.getnchannels() - print "nframes =", f.getnframes() - print "sampwidth =", f.getsampwidth() - print "framerate =", f.getframerate() - print "comptype =", f.getcomptype() - print "compname =", f.getcompname() - if sys.argv[2:]: - gn = sys.argv[2] - print "Writing", gn - g = open(gn, 'w') - g.setparams(f.getparams()) - while 1: - data = f.readframes(1024) - if not data: - break - g.writeframes(data) - g.close() + try: + print "Reading", fn + print "nchannels =", f.getnchannels() + print "nframes =", f.getnframes() + print "sampwidth =", f.getsampwidth() + print "framerate =", f.getframerate() + print "comptype =", f.getcomptype() + print "compname =", f.getcompname() + if sys.argv[2:]: + gn = sys.argv[2] + print "Writing", gn + g = open(gn, 'w') + try: + g.setparams(f.getparams()) + while 1: + data = f.readframes(1024) + if not data: + break + g.writeframes(data) + finally: + g.close() + print "Done." + finally: f.close() - print "Done." diff --git a/Misc/NEWS b/Misc/NEWS index c295fc39aca..f7e93723165 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -141,6 +141,8 @@ Library Tools/Demos ----------- +- Issue #18817: Fix a resource warning in Lib/aifc.py demo. + - Issue #18439: Make patchcheck work on Windows for ACKS, NEWS. - Issue #18448: Fix a typo in Demo/newmetaclasses/Eiffel.py. From 8b777a74fbd7b3263e5de00dff7cc80bec1862a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Sun, 25 Aug 2013 18:22:49 +0200 Subject: [PATCH 330/480] Issue #18763: subprocess: The file descriptors are now closed after calling the preexec_fn callback, which may open file descriptors. --- Lib/subprocess.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 50bec291c5a..814cfbe60f1 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -1247,16 +1247,17 @@ def _dup2(a, b): os.close(fd) closed.add(fd) - # Close all other fds, if asked for - if close_fds: - self._close_fds(but=errpipe_write) - if cwd is not None: os.chdir(cwd) if preexec_fn: preexec_fn() + # Close all other fds, if asked for - after + # preexec_fn(), which may open FDs. + if close_fds: + self._close_fds(but=errpipe_write) + if env is None: os.execvp(executable, args) else: From f6e6a71feabb1a8ffe4e444dc7730caea1c7db03 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 25 Aug 2013 23:56:43 +0300 Subject: [PATCH 331/480] #18833: add a test for test_telnetlib. Patch by Alex Volkov. --- Lib/test/test_telnetlib.py | 8 ++++++++ Misc/ACKS | 1 + 2 files changed, 9 insertions(+) diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py index 7fdb49e8e62..a5ed34bfa90 100644 --- a/Lib/test/test_telnetlib.py +++ b/Lib/test/test_telnetlib.py @@ -92,6 +92,14 @@ def testTimeoutOpen(self): self.assertEqual(telnet.sock.gettimeout(), 30) telnet.sock.close() + def testGetters(self): + # Test telnet getter methods + telnet = telnetlib.Telnet(HOST, self.port, timeout=30) + t_sock = telnet.sock + self.assertEqual(telnet.get_socket(), t_sock) + self.assertEqual(telnet.fileno(), t_sock.fileno()) + telnet.sock.close() + def _read_setUp(self): self.evt = threading.Event() self.dataq = Queue.Queue() diff --git a/Misc/ACKS b/Misc/ACKS index c6c2e2f2396..6a063966313 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1073,6 +1073,7 @@ Kurt Vile Norman Vine Frank Visser Johannes Vogel +Alex Volkov Martijn Vries Niki W. Waibel Wojtek Walczak From ecf0de26940c0d645108a1b22051443663412315 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Mon, 26 Aug 2013 01:31:30 +0300 Subject: [PATCH 332/480] =?UTF-8?q?#18803:=20fix=20more=20typos.=20=20Patc?= =?UTF-8?q?h=20by=20F=C3=A9vry=20Thibault.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Mac/BuildScript/build-installer.py | 4 ++-- Modules/unicodedata.c | 2 +- Parser/asdl_c.py | 2 +- Tools/freeze/checkextensions_win32.py | 4 ++-- Tools/freeze/makefreeze.py | 2 +- Tools/gdb/libpython.py | 2 +- Tools/i18n/msgfmt.py | 2 +- Tools/msi/msilib.py | 2 +- Tools/pybench/CommandLine.py | 2 +- Tools/pybench/systimes.py | 2 +- Tools/pynche/DetailsViewer.py | 2 +- Tools/scripts/fixnotice.py | 2 +- Tools/scripts/ifdef.py | 4 ++-- Tools/scripts/pathfix.py | 2 +- 14 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 004765ae580..ef822d58549 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -108,7 +108,7 @@ def getFullVersion(): ### There are some issues with the SDK selection below here, ### The resulting binary doesn't work on all platforms that ### it should. Always default to the 10.4u SDK until that -### isue is resolved. +### issue is resolved. ### ##if int(os.uname()[2].split('.')[0]) == 8: ## # Explicitly use the 10.4u (universal) SDK when @@ -1015,7 +1015,7 @@ def buildPython(): os.chdir(curdir) if PYTHON_3: - # Remove the 'Current' link, that way we don't accidently mess + # Remove the 'Current' link, that way we don't accidentally mess # with an already installed version of python 2 os.unlink(os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework', 'Versions', 'Current')) diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index edc811e0811..6f9c7e8e831 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -506,7 +506,7 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) stackptr = 0; isize = PyUnicode_GET_SIZE(input); - /* Overallocate atmost 10 characters. */ + /* Overallocate at most 10 characters. */ space = (isize > 10 ? 10 : isize) + isize; result = PyUnicode_FromUnicode(NULL, space); if (!result) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 666c48d199b..7ebc2369e38 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -977,7 +977,7 @@ def has_sequence(types, doing_specialization): class StaticVisitor(PickleVisitor): - CODE = '''Very simple, always emit this static code. Overide CODE''' + CODE = '''Very simple, always emit this static code. Override CODE''' def visit(self, object): self.emit(self.CODE, 0, reflow=False) diff --git a/Tools/freeze/checkextensions_win32.py b/Tools/freeze/checkextensions_win32.py index e5a8b29bb95..8c6444cf5ad 100644 --- a/Tools/freeze/checkextensions_win32.py +++ b/Tools/freeze/checkextensions_win32.py @@ -3,7 +3,7 @@ Under Windows it is unlikely the .obj files are of use, as special compiler options are needed (primarily to toggle the behavior of "public" symbols. -I dont consider it worth parsing the MSVC makefiles for compiler options. Even if +I don't consider it worth parsing the MSVC makefiles for compiler options. Even if we get it just right, a specific freeze application may have specific compiler options anyway (eg, to enable or disable specific functionality) @@ -14,7 +14,7 @@ your own). * This description can include: - The MSVC .dsp file for the extension. The .c source file names - are extraced from there. + are extracted from there. - Specific compiler/linker options - Flag to indicate if Unicode compilation is expected. diff --git a/Tools/freeze/makefreeze.py b/Tools/freeze/makefreeze.py index 1208b67fe07..c0f50566b0d 100644 --- a/Tools/freeze/makefreeze.py +++ b/Tools/freeze/makefreeze.py @@ -62,7 +62,7 @@ def makefreeze(base, dict, debug=0, entry_point=None, fail_import=()): outfp.write('\t{"%s", M_%s, %d},\n' % (mod, mangled, size)) outfp.write('\n') # The following modules have a NULL code pointer, indicating - # that the prozen program should not search for them on the host + # that the frozen program should not search for them on the host # system. Importing them will *always* raise an ImportError. # The zero value size is never used. for mod in fail_import: diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index e12769d7d27..b4bfdccc32c 100644 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -617,7 +617,7 @@ class PyDictObjectPtr(PyObjectPtr): def iteritems(self): ''' Yields a sequence of (PyObjectPtr key, PyObjectPtr value) pairs, - analagous to dict.iteritems() + analogous to dict.iteritems() ''' for i in safe_range(self.field('ma_mask') + 1): ep = self.field('ma_table') + i diff --git a/Tools/i18n/msgfmt.py b/Tools/i18n/msgfmt.py index 8c9b9dc9a34..2502a109c04 100755 --- a/Tools/i18n/msgfmt.py +++ b/Tools/i18n/msgfmt.py @@ -144,7 +144,7 @@ def make(filename, outfile): # This is a message with plural forms elif l.startswith('msgid_plural'): if section != ID: - print >> sys.stderr, 'msgid_plural not preceeded by msgid on %s:%d' %\ + print >> sys.stderr, 'msgid_plural not preceded by msgid on %s:%d' %\ (infile, lno) sys.exit(1) l = l[12:] diff --git a/Tools/msi/msilib.py b/Tools/msi/msilib.py index e02a3e2316f..760471c37b2 100644 --- a/Tools/msi/msilib.py +++ b/Tools/msi/msilib.py @@ -305,7 +305,7 @@ def init_database(name, schema, t.create(db) # Fill the validation table add_data(db, "_Validation", schema._Validation_records) - # Initialize the summary information, allowing atmost 20 properties + # Initialize the summary information, allowing at most 20 properties si = db.GetSummaryInformation(20) si.SetProperty(PID_TITLE, "Installation Database") si.SetProperty(PID_SUBJECT, ProductName) diff --git a/Tools/pybench/CommandLine.py b/Tools/pybench/CommandLine.py index 6601be5fb51..fde317841e1 100644 --- a/Tools/pybench/CommandLine.py +++ b/Tools/pybench/CommandLine.py @@ -458,7 +458,7 @@ def parse(self): handler = getattr(self, handlername) except AttributeError: if value == '': - # count the number of occurances + # count the number of occurrences if values.has_key(optionname): values[optionname] = values[optionname] + 1 else: diff --git a/Tools/pybench/systimes.py b/Tools/pybench/systimes.py index 013add278e9..db1210dee22 100644 --- a/Tools/pybench/systimes.py +++ b/Tools/pybench/systimes.py @@ -5,7 +5,7 @@ This module implements various different strategies for measuring performance timings. It tries to choose the best available method - based on the platforma and available tools. + based on the platform and available tools. On Windows, it is recommended to have the Mark Hammond win32 package installed. Alternatively, the Thomas Heller ctypes diff --git a/Tools/pynche/DetailsViewer.py b/Tools/pynche/DetailsViewer.py index 11a99a6afca..fb597b59fd8 100644 --- a/Tools/pynche/DetailsViewer.py +++ b/Tools/pynche/DetailsViewer.py @@ -26,7 +26,7 @@ other side. Thus if red were at 238 and 25 were added to it, red would have the value 7. - Preseve Distance + Preserve Distance When the increment or decrement would send any of the tied variations out of bounds, all tied variations are wrapped as one, so as to preserve the distance between them. Thus if green and blue were tied, diff --git a/Tools/scripts/fixnotice.py b/Tools/scripts/fixnotice.py index 0ae4872003a..e613b656722 100755 --- a/Tools/scripts/fixnotice.py +++ b/Tools/scripts/fixnotice.py @@ -2,7 +2,7 @@ """(Ostensibly) fix copyright notices in files. -Actually, this sript will simply replace a block of text in a file from one +Actually, this script will simply replace a block of text in a file from one string to another. It will only do this once though, i.e. not globally throughout the file. It writes a backup file and then does an os.rename() dance for atomicity. diff --git a/Tools/scripts/ifdef.py b/Tools/scripts/ifdef.py index 2ed7a6667bd..5487f1a74be 100755 --- a/Tools/scripts/ifdef.py +++ b/Tools/scripts/ifdef.py @@ -9,11 +9,11 @@ # options. On standard output it writes a copy of the input file(s) # minus those code sections that are suppressed by the selected # combination of defined/undefined symbols. The #if(n)def/#else/#else -# lines themselfs (if the #if(n)def tests for one of the mentioned +# lines themselves (if the #if(n)def tests for one of the mentioned # names) are removed as well. # Features: Arbitrary nesting of recognized and unrecognized -# preprocesor statements works correctly. Unrecognized #if* commands +# preprocessor statements works correctly. Unrecognized #if* commands # are left in place, so it will never remove too much, only too # little. It does accept whitespace around the '#' character. diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 7f6f191bbeb..850903a1000 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -135,7 +135,7 @@ def fix(filename): except os.error, msg: err('%s: rename failed (%r)\n' % (filename, msg)) return 1 - # Return succes + # Return success return 0 def fixline(line): From 1bb66779ec711a54f42f65fe3f2beb82798369cc Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Mon, 26 Aug 2013 14:00:39 +0300 Subject: [PATCH 333/480] #18839: document that sys.exit() will not accept a non-integer numeric value as exit status. --- Python/sysmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 814eccb1571..a868715ffcf 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -219,7 +219,7 @@ PyDoc_STRVAR(exit_doc, \n\ Exit the interpreter by raising SystemExit(status).\n\ If the status is omitted or None, it defaults to zero (i.e., success).\n\ -If the status is numeric, it will be used as the system exit status.\n\ +If the status is an integer, it will be used as the system exit status.\n\ If it is another kind of object, it will be printed and the system\n\ exit status will be one (i.e., failure)." ); From d78fe72ef4efd0cdc8562f512de8631dd75c4ccf Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 29 Aug 2013 10:34:23 +0300 Subject: [PATCH 334/480] Issue #18760: Improved cross-references in the xml package. --- Doc/library/xml.dom.minidom.rst | 2 +- Doc/library/xml.dom.rst | 9 ++++--- Doc/library/xml.etree.elementtree.rst | 5 ++-- Doc/library/xml.sax.handler.rst | 11 ++++++--- Doc/library/xml.sax.reader.rst | 32 +++++++++++++++--------- Doc/library/xml.sax.rst | 35 +++++++++++++++++---------- Doc/library/xml.sax.utils.rst | 17 +++++++------ 7 files changed, 68 insertions(+), 43 deletions(-) diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 70f0c030761..f6984a01112 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -58,7 +58,7 @@ instead: .. function:: parseString(string[, parser]) Return a :class:`Document` that represents the *string*. This method creates a - :class:`StringIO` object for the string and passes that on to :func:`parse`. + :class:`io.StringIO` object for the string and passes that on to :func:`parse`. Both functions return a :class:`Document` object representing the content of the document. diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index 565dc84907d..541d6496e18 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -441,14 +441,15 @@ objects: In addition, the Python DOM interface requires that some additional support is provided to allow :class:`NodeList` objects to be used as Python sequences. All -:class:`NodeList` implementations must include support for :meth:`__len__` and -:meth:`__getitem__`; this allows iteration over the :class:`NodeList` in +:class:`NodeList` implementations must include support for +:meth:`~object.__len__` and +:meth:`~object.__getitem__`; this allows iteration over the :class:`NodeList` in :keyword:`for` statements and proper support for the :func:`len` built-in function. If a DOM implementation supports modification of the document, the -:class:`NodeList` implementation must also support the :meth:`__setitem__` and -:meth:`__delitem__` methods. +:class:`NodeList` implementation must also support the +:meth:`~object.__setitem__` and :meth:`~object.__delitem__` methods. .. _dom-documenttype-objects: diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index e6ea004b577..fee56ff3fe9 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -680,8 +680,9 @@ Element Objects or contents. :class:`Element` objects also support the following sequence type methods - for working with subelements: :meth:`__delitem__`, :meth:`__getitem__`, - :meth:`__setitem__`, :meth:`__len__`. + for working with subelements: :meth:`~object.__delitem__`, + :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, + :meth:`~object.__len__`. Caution: Elements with no subelements will test as ``False``. This behavior will change in future versions. Use specific ``len(elem)`` or ``elem is diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst index 23f429e0732..15140a34549 100644 --- a/Doc/library/xml.sax.handler.rst +++ b/Doc/library/xml.sax.handler.rst @@ -240,7 +240,8 @@ events in the input document: Signals the start of an element in non-namespace mode. The *name* parameter contains the raw XML 1.0 name of the element type as a - string and the *attrs* parameter holds an object of the :class:`Attributes` + string and the *attrs* parameter holds an object of the + :class:`~xml.sax.xmlreader.Attributes` interface (see :ref:`attributes-objects`) containing the attributes of the element. The object passed as *attrs* may be re-used by the parser; holding on to a reference to it is not a reliable way to keep a copy of the attributes. @@ -263,7 +264,8 @@ events in the input document: The *name* parameter contains the name of the element type as a ``(uri, localname)`` tuple, the *qname* parameter contains the raw XML 1.0 name used in the source document, and the *attrs* parameter holds an instance of the - :class:`AttributesNS` interface (see :ref:`attributes-ns-objects`) + :class:`~xml.sax.xmlreader.AttributesNS` interface (see + :ref:`attributes-ns-objects`) containing the attributes of the element. If no namespace is associated with the element, the *uri* component of *name* will be ``None``. The object passed as *attrs* may be re-used by the parser; holding on to a reference to it is not @@ -379,8 +381,9 @@ ErrorHandler Objects -------------------- Objects with this interface are used to receive error and warning information -from the :class:`XMLReader`. If you create an object that implements this -interface, then register the object with your :class:`XMLReader`, the parser +from the :class:`~xml.sax.xmlreader.XMLReader`. If you create an object that +implements this interface, then register the object with your +:class:`~xml.sax.xmlreader.XMLReader`, the parser will call the methods in your object to report all warnings and errors. There are three levels of errors available: warnings, (possibly) recoverable errors, and unrecoverable errors. All methods take a :exc:`SAXParseException` as the diff --git a/Doc/library/xml.sax.reader.rst b/Doc/library/xml.sax.reader.rst index 4b3c18ac64c..e09ad7f811d 100644 --- a/Doc/library/xml.sax.reader.rst +++ b/Doc/library/xml.sax.reader.rst @@ -109,47 +109,50 @@ The :class:`XMLReader` interface supports the following methods: .. method:: XMLReader.getContentHandler() - Return the current :class:`ContentHandler`. + Return the current :class:`~xml.sax.handler.ContentHandler`. .. method:: XMLReader.setContentHandler(handler) - Set the current :class:`ContentHandler`. If no :class:`ContentHandler` is set, - content events will be discarded. + Set the current :class:`~xml.sax.handler.ContentHandler`. If no + :class:`~xml.sax.handler.ContentHandler` is set, content events will be + discarded. .. method:: XMLReader.getDTDHandler() - Return the current :class:`DTDHandler`. + Return the current :class:`~xml.sax.handler.DTDHandler`. .. method:: XMLReader.setDTDHandler(handler) - Set the current :class:`DTDHandler`. If no :class:`DTDHandler` is set, DTD + Set the current :class:`~xml.sax.handler.DTDHandler`. If no + :class:`~xml.sax.handler.DTDHandler` is set, DTD events will be discarded. .. method:: XMLReader.getEntityResolver() - Return the current :class:`EntityResolver`. + Return the current :class:`~xml.sax.handler.EntityResolver`. .. method:: XMLReader.setEntityResolver(handler) - Set the current :class:`EntityResolver`. If no :class:`EntityResolver` is set, + Set the current :class:`~xml.sax.handler.EntityResolver`. If no + :class:`~xml.sax.handler.EntityResolver` is set, attempts to resolve an external entity will result in opening the system identifier for the entity, and fail if it is not available. .. method:: XMLReader.getErrorHandler() - Return the current :class:`ErrorHandler`. + Return the current :class:`~xml.sax.handler.ErrorHandler`. .. method:: XMLReader.setErrorHandler(handler) - Set the current error handler. If no :class:`ErrorHandler` is set, errors will - be raised as exceptions, and warnings will be printed. + Set the current error handler. If no :class:`~xml.sax.handler.ErrorHandler` + is set, errors will be raised as exceptions, and warnings will be printed. .. method:: XMLReader.setLocale(locale) @@ -326,8 +329,13 @@ The :class:`Attributes` Interface --------------------------------- :class:`Attributes` objects implement a portion of the mapping protocol, -including the methods :meth:`copy`, :meth:`get`, :meth:`has_key`, :meth:`items`, -:meth:`keys`, and :meth:`values`. The following methods are also provided: +including the methods :meth:`~collections.abc.Mapping.copy`, +:meth:`~collections.abc.Mapping.get`, +:meth:`~collections.abc.Mapping.has_key`, +:meth:`~collections.abc.Mapping.items`, +:meth:`~collections.abc.Mapping.keys`, +and :meth:`~collections.abc.Mapping.values`. The following methods +are also provided: .. method:: Attributes.getLength() diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst index 75cfc119dde..25e4fa90324 100644 --- a/Doc/library/xml.sax.rst +++ b/Doc/library/xml.sax.rst @@ -29,7 +29,8 @@ The convenience functions are: .. function:: make_parser([parser_list]) - Create and return a SAX :class:`XMLReader` object. The first parser found will + Create and return a SAX :class:`~xml.sax.xmlreader.XMLReader` object. The + first parser found will be used. If *parser_list* is provided, it must be a sequence of strings which name modules that have a function named :func:`create_parser`. Modules listed in *parser_list* will be used before modules in the default list of parsers. @@ -39,8 +40,9 @@ The convenience functions are: Create a SAX parser and use it to parse a document. The document, passed in as *filename_or_stream*, can be a filename or a file object. The *handler* - parameter needs to be a SAX :class:`ContentHandler` instance. If - *error_handler* is given, it must be a SAX :class:`ErrorHandler` instance; if + parameter needs to be a SAX :class:`~handler.ContentHandler` instance. If + *error_handler* is given, it must be a SAX :class:`~handler.ErrorHandler` + instance; if omitted, :exc:`SAXParseException` will be raised on all errors. There is no return value; all work must be done by the *handler* passed in. @@ -65,10 +67,12 @@ For these objects, only the interfaces are relevant; they are normally not instantiated by the application itself. Since Python does not have an explicit notion of interface, they are formally introduced as classes, but applications may use implementations which do not inherit from the provided classes. The -:class:`InputSource`, :class:`Locator`, :class:`Attributes`, -:class:`AttributesNS`, and :class:`XMLReader` interfaces are defined in the +:class:`~xml.sax.xmlreader.InputSource`, :class:`~xml.sax.xmlreader.Locator`, +:class:`~xml.sax.xmlreader.Attributes`, :class:`~xml.sax.xmlreader.AttributesNS`, +and :class:`~xml.sax.xmlreader.XMLReader` interfaces are defined in the module :mod:`xml.sax.xmlreader`. The handler interfaces are defined in -:mod:`xml.sax.handler`. For convenience, :class:`InputSource` (which is often +:mod:`xml.sax.handler`. For convenience, +:class:`~xml.sax.xmlreader.InputSource` (which is often instantiated directly) and the handler classes are also available from :mod:`xml.sax`. These interfaces are described below. @@ -81,7 +85,8 @@ classes. Encapsulate an XML error or warning. This class can contain basic error or warning information from either the XML parser or the application: it can be subclassed to provide additional functionality or to add localization. Note - that although the handlers defined in the :class:`ErrorHandler` interface + that although the handlers defined in the + :class:`~xml.sax.handler.ErrorHandler` interface receive instances of this exception, it is not required to actually raise the exception --- it is also useful as a container for information. @@ -94,22 +99,26 @@ classes. .. exception:: SAXParseException(msg, exception, locator) - Subclass of :exc:`SAXException` raised on parse errors. Instances of this class - are passed to the methods of the SAX :class:`ErrorHandler` interface to provide - information about the parse error. This class supports the SAX :class:`Locator` - interface as well as the :class:`SAXException` interface. + Subclass of :exc:`SAXException` raised on parse errors. Instances of this + class are passed to the methods of the SAX + :class:`~xml.sax.handler.ErrorHandler` interface to provide information + about the parse error. This class supports the SAX + :class:`~xml.sax.xmlreader.Locator` interface as well as the + :class:`SAXException` interface. .. exception:: SAXNotRecognizedException(msg[, exception]) - Subclass of :exc:`SAXException` raised when a SAX :class:`XMLReader` is + Subclass of :exc:`SAXException` raised when a SAX + :class:`~xml.sax.xmlreader.XMLReader` is confronted with an unrecognized feature or property. SAX applications and extensions may use this class for similar purposes. .. exception:: SAXNotSupportedException(msg[, exception]) - Subclass of :exc:`SAXException` raised when a SAX :class:`XMLReader` is asked to + Subclass of :exc:`SAXException` raised when a SAX + :class:`~xml.sax.xmlreader.XMLReader` is asked to enable a feature that is not supported, or to set a property to a value that the implementation does not support. SAX applications and extensions may use this class for similar purposes. diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index c796ca8c86e..e774b6b8cb2 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -59,7 +59,8 @@ or as base classes. .. class:: XMLGenerator([out[, encoding]]) - This class implements the :class:`ContentHandler` interface by writing SAX + This class implements the :class:`~xml.sax.handler.ContentHandler` interface + by writing SAX events back into an XML document. In other words, using an :class:`XMLGenerator` as the content handler will reproduce the original document being parsed. *out* should be a file-like object which will default to *sys.stdout*. *encoding* is @@ -68,7 +69,8 @@ or as base classes. .. class:: XMLFilterBase(base) - This class is designed to sit between an :class:`XMLReader` and the client + This class is designed to sit between an + :class:`~xml.sax.xmlreader.XMLReader` and the client application's event handlers. By default, it does nothing but pass requests up to the reader and events on to the handlers unmodified, but subclasses can override specific methods to modify the event stream or the configuration @@ -77,9 +79,10 @@ or as base classes. .. function:: prepare_input_source(source[, base]) - This function takes an input source and an optional base URL and returns a fully - resolved :class:`InputSource` object ready for reading. The input source can be - given as a string, a file-like object, or an :class:`InputSource` object; - parsers will use this function to implement the polymorphic *source* argument to - their :meth:`parse` method. + This function takes an input source and an optional base URL and returns a + fully resolved :class:`~xml.sax.xmlreader.InputSource` object ready for + reading. The input source can be given as a string, a file-like object, or + an :class:`~xml.sax.xmlreader.InputSource` object; parsers will use this + function to implement the polymorphic *source* argument to their + :meth:`parse` method. From e5514561ca9fffb1f57f55cbebccb7feb1ba10d9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 29 Aug 2013 11:44:44 +0300 Subject: [PATCH 335/480] Issue #18743: Improved cross-references to the StringIO class. --- Doc/c-api/typeobj.rst | 2 +- Doc/library/gzip.rst | 6 +++--- Doc/library/mailbox.rst | 2 +- Doc/library/mimewriter.rst | 2 +- Doc/library/tempfile.rst | 2 +- Doc/library/xml.dom.minidom.rst | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index c35f7277072..bc596cca8fa 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -199,7 +199,7 @@ type objects) *must* have the :attr:`ob_size` field. An optional pointer to the instance print function. The print function is only called when the instance is printed to a *real* file; - when it is printed to a pseudo-file (like a :class:`StringIO` instance), the + when it is printed to a pseudo-file (like a :class:`~StringIO.StringIO` instance), the instance's :c:member:`~PyTypeObject.tp_repr` or :c:member:`~PyTypeObject.tp_str` function is called to convert it to a string. These are also called when the type's :c:member:`~PyTypeObject.tp_print` field is *NULL*. A type should never implement :c:member:`~PyTypeObject.tp_print` in a way that produces diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index ce11eec1cc2..4dd479ecf06 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -33,7 +33,7 @@ The module defines the following items: given a non-trivial value. The new class instance is based on *fileobj*, which can be a regular file, a - :class:`StringIO` object, or any other object which simulates a file. It + :class:`~StringIO.StringIO` object, or any other object which simulates a file. It defaults to ``None``, in which case *filename* is opened to provide a file object. @@ -65,9 +65,9 @@ The module defines the following items: Calling a :class:`GzipFile` object's :meth:`close` method does not close *fileobj*, since you might wish to append more material after the compressed - data. This also allows you to pass a :class:`StringIO` object opened for + data. This also allows you to pass a :class:`~StringIO.StringIO` object opened for writing as *fileobj*, and retrieve the resulting memory buffer using the - :class:`StringIO` object's :meth:`getvalue` method. + :class:`StringIO` object's :meth:`~StringIO.StringIO.getvalue` method. :class:`GzipFile` supports iteration and the :keyword:`with` statement. diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 3a87c13ff8a..c01656403ed 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -661,7 +661,7 @@ Maildir, mbox, MH, Babyl, and MMDF. In Babyl mailboxes, the headers of a message are not stored contiguously with the body of the message. To generate a file-like representation, the - headers and body are copied together into a :class:`StringIO` instance + headers and body are copied together into a :class:`~StringIO.StringIO` instance (from the :mod:`StringIO` module), which has an API identical to that of a file. As a result, the file-like object is truly independent of the underlying mailbox but does not save memory compared to a string diff --git a/Doc/library/mimewriter.rst b/Doc/library/mimewriter.rst index 2070ff614d7..a30caef4831 100644 --- a/Doc/library/mimewriter.rst +++ b/Doc/library/mimewriter.rst @@ -24,7 +24,7 @@ to rearrange their order. Return a new instance of the :class:`MimeWriter` class. The only argument passed, *fp*, is a file object to be used for writing. Note that a - :class:`StringIO` object could also be used. + :class:`~StringIO.StringIO` object could also be used. .. _mimewriter-objects: diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index ada647daa31..827f5f59baa 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -93,7 +93,7 @@ The module defines the following user-callable functions: causes the file to roll over to an on-disk file regardless of its size. The returned object is a file-like object whose :attr:`_file` attribute - is either a :class:`StringIO` object or a true file object, depending on + is either a :class:`~StringIO.StringIO` object or a true file object, depending on whether :func:`rollover` has been called. This file-like object can be used in a :keyword:`with` statement, just like a normal file. diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index f6984a01112..d4e363ac020 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -58,7 +58,7 @@ instead: .. function:: parseString(string[, parser]) Return a :class:`Document` that represents the *string*. This method creates a - :class:`io.StringIO` object for the string and passes that on to :func:`parse`. + :class:`~StringIO.StringIO` object for the string and passes that on to :func:`parse`. Both functions return a :class:`Document` object representing the content of the document. From 6c3a23a0a77f82bad00522e205134aafd03de0b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Thu, 29 Aug 2013 19:00:30 +0200 Subject: [PATCH 336/480] Issue #18643: Fix some test_socket failures due to large default socket buffer sizes. --- Lib/test/test_socket.py | 5 +++-- Lib/test/test_support.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 6e7218034d9..aef9b1a95d2 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -688,11 +688,12 @@ def raising_handler(*args): c.settimeout(1.5) with self.assertRaises(ZeroDivisionError): signal.alarm(1) - c.sendall(b"x" * (1024**2)) + c.sendall(b"x" * test_support.SOCK_MAX_SIZE) if with_timeout: signal.signal(signal.SIGALRM, ok_handler) signal.alarm(1) - self.assertRaises(socket.timeout, c.sendall, b"x" * (1024**2)) + self.assertRaises(socket.timeout, c.sendall, + b"x" * test_support.SOCK_MAX_SIZE) finally: signal.signal(signal.SIGALRM, old_alarm) c.close() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index be867bd507e..1f3c039b2f5 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -411,8 +411,14 @@ def fcmp(x, y): # fuzzy comparison function # Windows limit seems to be around 512 B, and many Unix kernels have a # 64 KiB pipe buffer size or 16 * PAGE_SIZE: take a few megs to be sure. # (see issue #17835 for a discussion of this number). -PIPE_MAX_SIZE = 4 *1024 * 1024 + 1 - +PIPE_MAX_SIZE = 4 * 1024 * 1024 + 1 + +# A constant likely larger than the underlying OS socket buffer size, to make +# writes blocking. +# The socket buffer sizes can usually be tuned system-wide (e.g. through sysctl +# on Linux), or on a per-socket basis (SO_SNDBUF/SO_RCVBUF). See issue #18643 +# for a discussion of this number). +SOCK_MAX_SIZE = 16 * 1024 * 1024 + 1 try: unicode From ead8cac0fa01c69f4ebf94ade9d1c6acbabe27b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Fri, 30 Aug 2013 23:30:50 +0200 Subject: [PATCH 337/480] Issue #18418: After fork(), reinit all threads states, not only active ones. Patch by A. Jesse Jiryu Davis. --- Lib/test/test_threading.py | 23 +++++++++++++++++++++++ Lib/threading.py | 2 +- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index ef04cd360a4..5a765f32083 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -443,6 +443,29 @@ def background_thread(evt): self.assertEqual(out, '') self.assertEqual(err, '') + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_is_alive_after_fork(self): + # Try hard to trigger #18418: is_alive() could sometimes be True on + # threads that vanished after a fork. + old_interval = sys.getcheckinterval() + + # Make the bug more likely to manifest. + sys.setcheckinterval(10) + + try: + for i in range(20): + t = threading.Thread(target=lambda: None) + t.start() + pid = os.fork() + if pid == 0: + os._exit(1 if t.is_alive() else 0) + else: + t.join() + pid, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + finally: + sys.setcheckinterval(old_interval) + class ThreadJoinOnShutdown(BaseTestCase): diff --git a/Lib/threading.py b/Lib/threading.py index 225448bf561..72c83197fd1 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1220,7 +1220,7 @@ def _after_fork(): new_active = {} current = current_thread() with _active_limbo_lock: - for thread in _active.itervalues(): + for thread in _enumerate(): # Any lock/condition variable may be currently locked or in an # invalid state, so we reinitialize them. if hasattr(thread, '_reset_internal_locks'): diff --git a/Misc/ACKS b/Misc/ACKS index 6a063966313..8977c02fa28 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -230,6 +230,7 @@ Scott David Daniels Ben Darnell Kushal Das Jonathan Dasteel +A. Jesse Jiryu Davis John DeGood Ned Deily Vincent Delft diff --git a/Misc/NEWS b/Misc/NEWS index f7e93723165..9544b3b9135 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #18418: After fork(), reinit all threads states, not only active ones. + Patch by A. Jesse Jiryu Davis. + - Issue #11973: Fix a problem in kevent. The flags and fflags fields are now properly handled as unsigned. From 985d35312373c7078783657be83814122927f084 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 30 Aug 2013 23:38:13 +0200 Subject: [PATCH 338/480] Issue #18851: Avoid a double close of subprocess pipes when the child process fails starting. --- Lib/subprocess.py | 73 +++++++++++++++++++++++-------------- Lib/test/test_subprocess.py | 38 ++++++++++++++++++- Misc/NEWS | 3 ++ 3 files changed, 85 insertions(+), 29 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 814cfbe60f1..3108537c504 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -698,12 +698,12 @@ def __init__(self, args, bufsize=0, executable=None, (p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite) = self._get_handles(stdin, stdout, stderr) + errread, errwrite), to_close = self._get_handles(stdin, stdout, stderr) try: self._execute_child(args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, - startupinfo, creationflags, shell, + startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) @@ -711,18 +711,12 @@ def __init__(self, args, bufsize=0, executable=None, # Preserve original exception in case os.close raises. exc_type, exc_value, exc_trace = sys.exc_info() - to_close = [] - # Only close the pipes we created. - if stdin == PIPE: - to_close.extend((p2cread, p2cwrite)) - if stdout == PIPE: - to_close.extend((c2pread, c2pwrite)) - if stderr == PIPE: - to_close.extend((errread, errwrite)) - for fd in to_close: try: - os.close(fd) + if mswindows: + fd.Close() + else: + os.close(fd) except EnvironmentError: pass @@ -816,8 +810,9 @@ def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ + to_close = set() if stdin is None and stdout is None and stderr is None: - return (None, None, None, None, None, None) + return (None, None, None, None, None, None), to_close p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None @@ -835,6 +830,10 @@ def _get_handles(self, stdin, stdout, stderr): # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) p2cread = self._make_inheritable(p2cread) + # We just duplicated the handle, it has to be closed at the end + to_close.add(p2cread) + if stdin == PIPE: + to_close.add(p2cwrite) if stdout is None: c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE) @@ -848,6 +847,10 @@ def _get_handles(self, stdin, stdout, stderr): # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) c2pwrite = self._make_inheritable(c2pwrite) + # We just duplicated the handle, it has to be closed at the end + to_close.add(c2pwrite) + if stdout == PIPE: + to_close.add(c2pread) if stderr is None: errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE) @@ -863,10 +866,14 @@ def _get_handles(self, stdin, stdout, stderr): # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) errwrite = self._make_inheritable(errwrite) + # We just duplicated the handle, it has to be closed at the end + to_close.add(errwrite) + if stderr == PIPE: + to_close.add(errread) return (p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite) + errread, errwrite), to_close def _make_inheritable(self, handle): @@ -895,7 +902,7 @@ def _find_w9xpopen(self): def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, - startupinfo, creationflags, shell, + startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): @@ -934,6 +941,10 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, # kill children. creationflags |= _subprocess.CREATE_NEW_CONSOLE + def _close_in_parent(fd): + fd.Close() + to_close.remove(fd) + # Start the process try: hp, ht, pid, tid = _subprocess.CreateProcess(executable, args, @@ -958,11 +969,11 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, # pipe will not close when the child process exits and the # ReadFile will hang. if p2cread is not None: - p2cread.Close() + _close_in_parent(p2cread) if c2pwrite is not None: - c2pwrite.Close() + _close_in_parent(c2pwrite) if errwrite is not None: - errwrite.Close() + _close_in_parent(errwrite) # Retain the process handle, but close the thread handle self._child_created = True @@ -1088,6 +1099,7 @@ def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ + to_close = set() p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None @@ -1096,6 +1108,7 @@ def _get_handles(self, stdin, stdout, stderr): pass elif stdin == PIPE: p2cread, p2cwrite = self.pipe_cloexec() + to_close.update((p2cread, p2cwrite)) elif isinstance(stdin, int): p2cread = stdin else: @@ -1106,6 +1119,7 @@ def _get_handles(self, stdin, stdout, stderr): pass elif stdout == PIPE: c2pread, c2pwrite = self.pipe_cloexec() + to_close.update((c2pread, c2pwrite)) elif isinstance(stdout, int): c2pwrite = stdout else: @@ -1116,6 +1130,7 @@ def _get_handles(self, stdin, stdout, stderr): pass elif stderr == PIPE: errread, errwrite = self.pipe_cloexec() + to_close.update((errread, errwrite)) elif stderr == STDOUT: errwrite = c2pwrite elif isinstance(stderr, int): @@ -1126,7 +1141,7 @@ def _get_handles(self, stdin, stdout, stderr): return (p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite) + errread, errwrite), to_close def _set_cloexec_flag(self, fd, cloexec=True): @@ -1170,7 +1185,7 @@ def _close_fds(self, but): def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, - startupinfo, creationflags, shell, + startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): @@ -1189,6 +1204,10 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, if executable is None: executable = args[0] + def _close_in_parent(fd): + os.close(fd) + to_close.remove(fd) + # For transferring possible exec failure from child to parent # The first char specifies the exception type: 0 means # OSError, 1 means some other error. @@ -1283,17 +1302,17 @@ def _dup2(a, b): # be sure the FD is closed no matter what os.close(errpipe_write) - if p2cread is not None and p2cwrite is not None: - os.close(p2cread) - if c2pwrite is not None and c2pread is not None: - os.close(c2pwrite) - if errwrite is not None and errread is not None: - os.close(errwrite) - # Wait for exec to fail or succeed; possibly raising exception # Exception limited to 1M data = _eintr_retry_call(os.read, errpipe_read, 1048576) finally: + if p2cread is not None and p2cwrite is not None: + _close_in_parent(p2cread) + if c2pwrite is not None and c2pread is not None: + _close_in_parent(c2pwrite) + if errwrite is not None and errread is not None: + _close_in_parent(errwrite) + # be sure the FD is closed no matter what os.close(errpipe_read) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index f43b51caf83..0efcdbf2534 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -14,6 +14,10 @@ import resource except ImportError: resource = None +try: + import threading +except ImportError: + threading = None mswindows = (sys.platform == "win32") @@ -629,6 +633,36 @@ def test_leaking_fds_on_error(self): if c.exception.errno not in (errno.ENOENT, errno.EACCES): raise c.exception + @unittest.skipIf(threading is None, "threading required") + def test_double_close_on_error(self): + # Issue #18851 + fds = [] + def open_fds(): + for i in range(20): + fds.extend(os.pipe()) + time.sleep(0.001) + t = threading.Thread(target=open_fds) + t.start() + try: + with self.assertRaises(EnvironmentError): + subprocess.Popen(['nonexisting_i_hope'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + t.join() + exc = None + for fd in fds: + # If a double close occurred, some of those fds will + # already have been closed by mistake, and os.close() + # here will raise. + try: + os.close(fd) + except OSError as e: + exc = e + if exc is not None: + raise exc + def test_handles_closed_on_exception(self): # If CreateProcess exits with an error, ensure the # duplicate output handles are released @@ -783,7 +817,7 @@ def __init__(self, testcase, *args, **kwargs): def _execute_child( self, args, executable, preexec_fn, close_fds, cwd, env, - universal_newlines, startupinfo, creationflags, shell, + universal_newlines, startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): @@ -791,7 +825,7 @@ def _execute_child( subprocess.Popen._execute_child( self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, - startupinfo, creationflags, shell, + startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) diff --git a/Misc/NEWS b/Misc/NEWS index 9544b3b9135..0db727f9320 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #18851: Avoid a double close of subprocess pipes when the child + process fails starting. + - Issue #18418: After fork(), reinit all threads states, not only active ones. Patch by A. Jesse Jiryu Davis. From 407dbcf3b9b3a8de55aafdd2347408f1e5f356e2 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 31 Aug 2013 16:27:08 -0400 Subject: [PATCH 339/480] Issue #18489: Add complete, gui-free tests for idlelib.SearchEngine. Patch import and initialization in SearchEngine to make testing easier. Improve docstrings, especially to clarify the double role of 'ok' parameters. Original patch by Phil Webster. --- Lib/idlelib/SearchEngine.py | 72 ++--- Lib/idlelib/idle_test/test_searchengine.py | 326 +++++++++++++++++++++ Misc/NEWS | 2 + 3 files changed, 366 insertions(+), 34 deletions(-) create mode 100644 Lib/idlelib/idle_test/test_searchengine.py diff --git a/Lib/idlelib/SearchEngine.py b/Lib/idlelib/SearchEngine.py index 961a9218eee..ea6f1dede69 100644 --- a/Lib/idlelib/SearchEngine.py +++ b/Lib/idlelib/SearchEngine.py @@ -1,6 +1,6 @@ '''Define SearchEngine for search dialogs.''' import re -from Tkinter import * +from Tkinter import StringVar, BooleanVar, TclError import tkMessageBox def get(root): @@ -22,14 +22,13 @@ def __init__(self, root): The dialogs bind these to the UI elements present in the dialogs. ''' - self.root = root - self.patvar = StringVar(root) # search pattern - self.revar = BooleanVar(root) # regular expression? - self.casevar = BooleanVar(root) # match case? - self.wordvar = BooleanVar(root) # match whole word? - self.wrapvar = BooleanVar(root) # wrap around buffer? - self.wrapvar.set(1) # (on by default) - self.backvar = BooleanVar(root) # search backwards? + self.root = root # need for report_error() + self.patvar = StringVar(root, '') # search pattern + self.revar = BooleanVar(root, False) # regular expression? + self.casevar = BooleanVar(root, False) # match case? + self.wordvar = BooleanVar(root, False) # match whole word? + self.wrapvar = BooleanVar(root, True) # wrap around buffer? + self.backvar = BooleanVar(root, False) # search backwards? # Access methods @@ -56,9 +55,16 @@ def isback(self): # Higher level access methods + def setcookedpat(self, pat): + "Set pattern after escaping if re." + # called only in SearchDialog.py: 66 + if self.isre(): + pat = re.escape(pat) + self.setpat(pat) + def getcookedpat(self): pat = self.getpat() - if not self.isre(): + if not self.isre(): # if True, see setcookedpat pat = re.escape(pat) if self.isword(): pat = r"\b%s\b" % pat @@ -90,33 +96,28 @@ def report_error(self, pat, msg, col=-1): # Derived class could override this with something fancier msg = "Error: " + str(msg) if pat: - msg = msg + "\np\Pattern: " + str(pat) + msg = msg + "\nPattern: " + str(pat) if col >= 0: msg = msg + "\nOffset: " + str(col) tkMessageBox.showerror("Regular expression error", msg, master=self.root) - def setcookedpat(self, pat): - if self.isre(): - pat = re.escape(pat) - self.setpat(pat) - def search_text(self, text, prog=None, ok=0): - '''Return (lineno, matchobj) for prog in text widget, or None. + '''Return (lineno, matchobj) or None for forward/backward search. + + This function calls the right function with the right arguments. + It directly return the result of that call. - If prog is given, it should be a precompiled pattern. - Wrap (yes/no) and direction (forward/back) settings are used. + Text is a text widget. Prog is a precompiled pattern. + The ok parameteris a bit complicated as it has two effects. - The search starts at the selection (if there is one) or at the - insert mark (otherwise). If the search is forward, it starts - at the right of the selection; for a backward search, it - starts at the left end. An empty match exactly at either end - of the selection (or at the insert mark if there is no - selection) is ignored unless the ok flag is true -- this is - done to guarantee progress. + If there is a selection, the search begin at either end, + depending on the direction setting and ok, with ok meaning that + the search starts with the selection. Otherwise, search begins + at the insert mark. - If the search is allowed to wrap around, it will return the - original selection if (and only if) it is the only match. + To aid progress, the search functions do not return an empty + match at the starting position unless ok is True. ''' if not prog: @@ -188,15 +189,18 @@ def search_backward(self, text, prog, line, col, wrap, ok=0): return None def search_reverse(prog, chars, col): - '''Search backwards in a string (line of text). + '''Search backwards and return an re match object or None. This is done by searching forwards until there is no match. + Prog: compiled re object with a search method returning a match. + Chars: line of text, without \n. + Col: stop index for the search; the limit for match.end(). ''' m = prog.search(chars) if not m: return None found = None - i, j = m.span() + i, j = m.span() # m.start(), m.end() == match slice indexes while i < col and j <= col: found = m if i == j: @@ -226,7 +230,7 @@ def get_line_col(index): line, col = map(int, index.split(".")) # Fails on invalid index return line, col -##if __name__ == "__main__": -## from test import support; support.use_resources = ['gui'] -## import unittest -## unittest.main('idlelib.idle_test.test_searchengine', verbosity=2, exit=False) +if __name__ == "__main__": + from test import support; support.use_resources = ['gui'] + import unittest + unittest.main('idlelib.idle_test.test_searchengine', verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_searchengine.py b/Lib/idlelib/idle_test/test_searchengine.py new file mode 100644 index 00000000000..e5e720de956 --- /dev/null +++ b/Lib/idlelib/idle_test/test_searchengine.py @@ -0,0 +1,326 @@ +'''Test functions and SearchEngine class in SearchEngine.py.''' + +# With mock replacements, the module does not use any gui widgets. +# The use of tk.Text is avoided (for now, until mock Text is improved) +# by patching instances with an index function returning what is needed. +# This works because mock Text.get does not use .index. + +import re +import unittest +from test.test_support import requires +from Tkinter import BooleanVar, StringVar, TclError # ,Tk, Text +import tkMessageBox +from idlelib import SearchEngine as se +from idlelib.idle_test.mock_tk import Var, Mbox +from idlelib.idle_test.mock_tk import Text as mockText + +def setUpModule(): + # Replace s-e module tkinter imports other than non-gui TclError. + se.BooleanVar = Var + se.StringVar = Var + se.tkMessageBox = Mbox + +def tearDownModule(): + # Restore 'just in case', though other tests should also replace. + se.BooleanVar = BooleanVar + se.StringVar = StringVar + se.tkMessageBox = tkMessageBox + + +class Mock: + def __init__(self, *args, **kwargs): pass + +class GetTest(unittest.TestCase): + # SearchEngine.get returns singleton created & saved on first call. + def test_get(self): + saved_Engine = se.SearchEngine + se.SearchEngine = Mock # monkey-patch class + try: + root = Mock() + engine = se.get(root) + self.assertIsInstance(engine, se.SearchEngine) + self.assertIs(root._searchengine, engine) + self.assertIs(se.get(root), engine) + finally: + se.SearchEngine = saved_Engine # restore class to module + +class GetLineColTest(unittest.TestCase): + # Test simple text-independent helper function + def test_get_line_col(self): + self.assertEqual(se.get_line_col('1.0'), (1, 0)) + self.assertEqual(se.get_line_col('1.11'), (1, 11)) + + self.assertRaises(ValueError, se.get_line_col, ('1.0 lineend')) + self.assertRaises(ValueError, se.get_line_col, ('end')) + +class GetSelectionTest(unittest.TestCase): + # Test text-dependent helper function. +## # Need gui for text.index('sel.first/sel.last/insert'). +## @classmethod +## def setUpClass(cls): +## requires('gui') +## cls.root = Tk() +## +## @classmethod +## def tearDownClass(cls): +## cls.root.destroy() + + def test_get_selection(self): + # text = Text(master=self.root) + text = mockText() + text.insert('1.0', 'Hello World!') + + # fix text.index result when called in get_selection + def sel(s): + # select entire text, cursor irrelevant + if s == 'sel.first': return '1.0' + if s == 'sel.last': return '1.12' + raise TclError + text.index = sel # replaces .tag_add('sel', '1.0, '1.12') + self.assertEqual(se.get_selection(text), ('1.0', '1.12')) + + def mark(s): + # no selection, cursor after 'Hello' + if s == 'insert': return '1.5' + raise TclError + text.index = mark # replaces .mark_set('insert', '1.5') + self.assertEqual(se.get_selection(text), ('1.5', '1.5')) + + +class ReverseSearchTest(unittest.TestCase): + # Test helper function that searches backwards within a line. + def test_search_reverse(self): + Equal = self.assertEqual + line = "Here is an 'is' test text." + prog = re.compile('is') + Equal(se.search_reverse(prog, line, len(line)).span(), (12, 14)) + Equal(se.search_reverse(prog, line, 14).span(), (12, 14)) + Equal(se.search_reverse(prog, line, 13).span(), (5, 7)) + Equal(se.search_reverse(prog, line, 7).span(), (5, 7)) + Equal(se.search_reverse(prog, line, 6), None) + + +class SearchEngineTest(unittest.TestCase): + # Test class methods that do not use Text widget. + + def setUp(self): + self.engine = se.SearchEngine(root=None) + # Engine.root is only used to create error message boxes. + # The mock replacement ignores the root argument. + + def test_is_get(self): + engine = self.engine + Equal = self.assertEqual + + Equal(engine.getpat(), '') + engine.setpat('hello') + Equal(engine.getpat(), 'hello') + + Equal(engine.isre(), False) + engine.revar.set(1) + Equal(engine.isre(), True) + + Equal(engine.iscase(), False) + engine.casevar.set(1) + Equal(engine.iscase(), True) + + Equal(engine.isword(), False) + engine.wordvar.set(1) + Equal(engine.isword(), True) + + Equal(engine.iswrap(), True) + engine.wrapvar.set(0) + Equal(engine.iswrap(), False) + + Equal(engine.isback(), False) + engine.backvar.set(1) + Equal(engine.isback(), True) + + def test_setcookedpat(self): + engine = self.engine + engine.setcookedpat('\s') + self.assertEqual(engine.getpat(), '\s') + engine.revar.set(1) + engine.setcookedpat('\s') + self.assertEqual(engine.getpat(), r'\\s') + + def test_getcookedpat(self): + engine = self.engine + Equal = self.assertEqual + + Equal(engine.getcookedpat(), '') + engine.setpat('hello') + Equal(engine.getcookedpat(), 'hello') + engine.wordvar.set(True) + Equal(engine.getcookedpat(), r'\bhello\b') + engine.wordvar.set(False) + + engine.setpat('\s') + Equal(engine.getcookedpat(), r'\\s') + engine.revar.set(True) + Equal(engine.getcookedpat(), '\s') + + def test_getprog(self): + engine = self.engine + Equal = self.assertEqual + + engine.setpat('Hello') + temppat = engine.getprog() + Equal(temppat.pattern, re.compile('Hello', re.IGNORECASE).pattern) + engine.casevar.set(1) + temppat = engine.getprog() + Equal(temppat.pattern, re.compile('Hello').pattern, 0) + + engine.setpat('') + Equal(engine.getprog(), None) + engine.setpat('+') + engine.revar.set(1) + Equal(engine.getprog(), None) + self.assertEqual(Mbox.showerror.message, + 'Error: nothing to repeat\nPattern: +') + + def test_report_error(self): + showerror = Mbox.showerror + Equal = self.assertEqual + pat = '[a-z' + msg = 'unexpected end of regular expression' + + Equal(self.engine.report_error(pat, msg), None) + Equal(showerror.title, 'Regular expression error') + expected_message = ("Error: " + msg + "\nPattern: [a-z") + Equal(showerror.message, expected_message) + + Equal(self.engine.report_error(pat, msg, 5), None) + Equal(showerror.title, 'Regular expression error') + expected_message += "\nOffset: 5" + Equal(showerror.message, expected_message) + + +class SearchTest(unittest.TestCase): + # Test that search_text makes right call to right method. + + @classmethod + def setUpClass(cls): +## requires('gui') +## cls.root = Tk() +## cls.text = Text(master=cls.root) + cls.text = mockText() + test_text = ( + 'First line\n' + 'Line with target\n' + 'Last line\n') + cls.text.insert('1.0', test_text) + cls.pat = re.compile('target') + + cls.engine = se.SearchEngine(None) + cls.engine.search_forward = lambda *args: ('f', args) + cls.engine.search_backward = lambda *args: ('b', args) + +## @classmethod +## def tearDownClass(cls): +## cls.root.destroy() + + def test_search(self): + Equal = self.assertEqual + engine = self.engine + search = engine.search_text + text = self.text + pat = self.pat + + engine.patvar.set(None) + #engine.revar.set(pat) + Equal(search(text), None) + + def mark(s): + # no selection, cursor after 'Hello' + if s == 'insert': return '1.5' + raise TclError + text.index = mark + Equal(search(text, pat), ('f', (text, pat, 1, 5, True, False))) + engine.wrapvar.set(False) + Equal(search(text, pat), ('f', (text, pat, 1, 5, False, False))) + engine.wrapvar.set(True) + engine.backvar.set(True) + Equal(search(text, pat), ('b', (text, pat, 1, 5, True, False))) + engine.backvar.set(False) + + def sel(s): + if s == 'sel.first': return '2.10' + if s == 'sel.last': return '2.16' + raise TclError + text.index = sel + Equal(search(text, pat), ('f', (text, pat, 2, 16, True, False))) + Equal(search(text, pat, True), ('f', (text, pat, 2, 10, True, True))) + engine.backvar.set(True) + Equal(search(text, pat), ('b', (text, pat, 2, 10, True, False))) + Equal(search(text, pat, True), ('b', (text, pat, 2, 16, True, True))) + + +class ForwardBackwardTest(unittest.TestCase): + # Test that search_forward method finds the target. +## @classmethod +## def tearDownClass(cls): +## cls.root.destroy() + + @classmethod + def setUpClass(cls): + cls.engine = se.SearchEngine(None) +## requires('gui') +## cls.root = Tk() +## cls.text = Text(master=cls.root) + cls.text = mockText() + # search_backward calls index('end-1c') + cls.text.index = lambda index: '4.0' + test_text = ( + 'First line\n' + 'Line with target\n' + 'Last line\n') + cls.text.insert('1.0', test_text) + cls.pat = re.compile('target') + cls.res = (2, (10, 16)) # line, slice indexes of 'target' + cls.failpat = re.compile('xyz') # not in text + cls.emptypat = re.compile('\w*') # empty match possible + + def make_search(self, func): + def search(pat, line, col, wrap, ok=0): + res = func(self.text, pat, line, col, wrap, ok) + # res is (line, matchobject) or None + return (res[0], res[1].span()) if res else res + return search + + def test_search_forward(self): + # search for non-empty match + Equal = self.assertEqual + forward = self.make_search(self.engine.search_forward) + pat = self.pat + Equal(forward(pat, 1, 0, True), self.res) + Equal(forward(pat, 3, 0, True), self.res) # wrap + Equal(forward(pat, 3, 0, False), None) # no wrap + Equal(forward(pat, 2, 10, False), self.res) + + Equal(forward(self.failpat, 1, 0, True), None) + Equal(forward(self.emptypat, 2, 9, True, ok=True), (2, (9, 9))) + #Equal(forward(self.emptypat, 2, 9, True), self.res) + # While the initial empty match is correctly ignored, skipping + # the rest of the line and returning (3, (0,4)) seems buggy - tjr. + Equal(forward(self.emptypat, 2, 10, True), self.res) + + def test_search_backward(self): + # search for non-empty match + Equal = self.assertEqual + backward = self.make_search(self.engine.search_backward) + pat = self.pat + Equal(backward(pat, 3, 5, True), self.res) + Equal(backward(pat, 2, 0, True), self.res) # wrap + Equal(backward(pat, 2, 0, False), None) # no wrap + Equal(backward(pat, 2, 16, False), self.res) + + Equal(backward(self.failpat, 3, 9, True), None) + Equal(backward(self.emptypat, 2, 10, True, ok=True), (2, (9,9))) + # Accepted because 9 < 10, not because ok=True. + # It is not clear that ok=True is useful going back - tjr + Equal(backward(self.emptypat, 2, 9, True), (2, (5, 9))) + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) diff --git a/Misc/NEWS b/Misc/NEWS index 0db727f9320..afb7464be0f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -171,6 +171,8 @@ Build IDLE ---- +- Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster. + - Issue #18429: Format / Format Paragraph, now works when comment blocks are selected. As with text blocks, this works best when the selection only includes complete lines. From 4140430362b6c153f20a0349e47a6b076874f658 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 31 Aug 2013 20:37:21 -0400 Subject: [PATCH 340/480] Silence deprecation warning in test_unittest for 'None > 1'. This is the same change that was made in 3.x when this became an error. --- Lib/unittest/test/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py index d1cefae4c17..3b661d33408 100644 --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -159,7 +159,7 @@ class ATextResult(unittest.TextTestResult, AResult): # This used to raise an exception due to TextTestResult not passing # on arguments in its __init__ super call - ATextResult(None, None, None) + ATextResult(None, None, 1) def testBufferAndFailfast(self): From 4b7ce7a5ca75cf419df8b9978a897d9f46efc55d Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 31 Aug 2013 21:14:00 -0400 Subject: [PATCH 341/480] Silence deprecation warning in test_mmap for ValueError.message. Replacement is from 3.3, with 'p' added to make '...Regexp'. --- Lib/test/test_mmap.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index f84b20aa441..9fd218418ce 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -470,14 +470,10 @@ def test_empty_file (self): f = open (TESTFN, 'w+b') f.close() with open(TESTFN, "rb") as f : - try: - m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) - m.close() - self.fail("should not have been able to mmap empty file") - except ValueError as e: - self.assertEqual(e.message, "cannot mmap an empty file") - except: - self.fail("unexpected exception: " + str(e)) + self.assertRaisesRegexp(ValueError, + "cannot mmap an empty file", + mmap.mmap, f.fileno(), 0, + access=mmap.ACCESS_READ) def test_offset (self): f = open (TESTFN, 'w+b') From ff55c007fd576e568869abb908a7d47f9312db98 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 1 Sep 2013 19:06:35 -0400 Subject: [PATCH 342/480] document that various functions that parse from source will interpret things as latin-1 (closes #18870) --- Doc/library/ast.rst | 8 ++++---- Doc/library/functions.rst | 12 +++++++----- Doc/reference/simple_stmts.rst | 19 ++++++++++--------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 5130d0045d5..4db40609365 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -131,10 +131,10 @@ and classes for traversing abstract syntax trees: .. function:: literal_eval(node_or_string) - Safely evaluate an expression node or a string containing a Python - expression. The string or node provided may only consist of the following - Python literal structures: strings, numbers, tuples, lists, dicts, booleans, - and ``None``. + Safely evaluate an expression node or a Unicode or *Latin-1* encoded string + containing a Python expression. The string or node provided may only consist + of the following Python literal structures: strings, numbers, tuples, lists, + dicts, booleans, and ``None``. This can be used for safely evaluating strings containing Python expressions from untrusted sources without the need to parse the values oneself. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 08fb7fedcc5..d40452adbbc 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -199,8 +199,10 @@ available. They are listed here in alphabetical order. Compile the *source* into a code or AST object. Code objects can be executed by an :keyword:`exec` statement or evaluated by a call to :func:`eval`. - *source* can either be a string or an AST object. Refer to the :mod:`ast` - module documentation for information on how to work with AST objects. + *source* can either be a Unicode string, a *Latin-1* encoded string or an + AST object. + Refer to the :mod:`ast` module documentation for information on how to work + with AST objects. The *filename* argument should give the file from which the code was read; pass some recognizable value if it wasn't read from a file (``''`` is @@ -388,9 +390,9 @@ available. They are listed here in alphabetical order. .. function:: eval(expression[, globals[, locals]]) - The arguments are a string and optional globals and locals. If provided, - *globals* must be a dictionary. If provided, *locals* can be any mapping - object. + The arguments are a Unicode or *Latin-1* encoded string and optional + globals and locals. If provided, *globals* must be a dictionary. + If provided, *locals* can be any mapping object. .. versionchanged:: 2.4 formerly *locals* was required to be a dictionary. diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index a6fd0649a91..254d1ba1147 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -981,15 +981,16 @@ The :keyword:`exec` statement exec_stmt: "exec" `or_expr` ["in" `expression` ["," `expression`]] This statement supports dynamic execution of Python code. The first expression -should evaluate to either a string, an open file object, a code object, or a -tuple. If it is a string, the string is parsed as a suite of Python statements -which is then executed (unless a syntax error occurs). [#]_ If it is an open -file, the file is parsed until EOF and executed. If it is a code object, it is -simply executed. For the interpretation of a tuple, see below. In all cases, -the code that's executed is expected to be valid as file input (see section -:ref:`file-input`). Be aware that the :keyword:`return` and :keyword:`yield` -statements may not be used outside of function definitions even within the -context of code passed to the :keyword:`exec` statement. +should evaluate to either a Unicode string, a *Latin-1* encoded string, an open +file object, a code object, or a tuple. If it is a string, the string is parsed +as a suite of Python statements which is then executed (unless a syntax error +occurs). [#]_ If it is an open file, the file is parsed until EOF and executed. +If it is a code object, it is simply executed. For the interpretation of a +tuple, see below. In all cases, the code that's executed is expected to be +valid as file input (see section :ref:`file-input`). Be aware that the +:keyword:`return` and :keyword:`yield` statements may not be used outside of +function definitions even within the context of code passed to the +:keyword:`exec` statement. In all cases, if the optional parts are omitted, the code is executed in the current scope. If only the first expression after ``in`` is specified, From 462779950ff16260684452a436ce4de2ac28e55c Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 3 Sep 2013 14:47:00 +0200 Subject: [PATCH 343/480] Python 2.6's ssl module has neither OPENSSL_VERSION_INFO nor _OPENSSL_API_VERSION --- Lib/test/test_ssl.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index f779dbae226..59294b45479 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -102,21 +102,12 @@ def test_parse_cert_CVE_2013_4073(self): (('emailAddress', 'python-dev@python.org'),)) self.assertEqual(p['subject'], subject) self.assertEqual(p['issuer'], subject) - if ssl._OPENSSL_API_VERSION >= (0, 9, 8): - san = (('DNS', 'altnull.python.org\x00example.com'), - ('email', 'null@python.org\x00user@example.org'), - ('URI', 'http://null.python.org\x00http://example.org'), - ('IP Address', '192.0.2.1'), - ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) - else: - # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName - san = (('DNS', 'altnull.python.org\x00example.com'), - ('email', 'null@python.org\x00user@example.org'), - ('URI', 'http://null.python.org\x00http://example.org'), - ('IP Address', '192.0.2.1'), - ('IP Address', '')) - - self.assertEqual(p['subjectAltName'], san) + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1')) + + self.assertEqual(p['subjectAltName'][:4], san) def test_DER_to_PEM(self): with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f: From 561caf343f18b1b777432d9063b3b1010adbdcb8 Mon Sep 17 00:00:00 2001 From: Eli Bendersky Date: Tue, 3 Sep 2013 06:41:58 -0700 Subject: [PATCH 344/480] Close #18912: Fix indentation in docstring Contributed by Jeroen Van Goey --- Modules/itertoolsmodule.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ea822913662..c0456b839f7 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -3426,10 +3426,10 @@ PyDoc_STRVAR(count_doc, Return a count object whose .next() method returns consecutive values.\n\ Equivalent to:\n\n\ def count(firstval=0, step=1):\n\ - x = firstval\n\ - while 1:\n\ - yield x\n\ - x += step\n"); + x = firstval\n\ + while 1:\n\ + yield x\n\ + x += step\n"); static PyTypeObject count_type = { PyVarObject_HEAD_INIT(NULL, 0) From 03cec5311aa6f37c6969c925d409adab5a29c0a6 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 4 Sep 2013 20:46:33 +0200 Subject: [PATCH 345/480] Issue #18876: The FileIO.mode attribute now better reflects the actual mode under which the file was opened. Patch by Erik Bray. --- Lib/test/test_fileio.py | 17 ++++++++++++++++- Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/_io/fileio.c | 23 +++++++++++++---------- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index b74cec24f79..e7955cce3ee 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -305,7 +305,7 @@ def testAbles(self): finally: os.unlink(TESTFN) - def testModeStrings(self): + def testInvalidModeStrings(self): # check invalid mode strings for mode in ("", "aU", "wU+", "rw", "rt"): try: @@ -316,6 +316,21 @@ def testModeStrings(self): f.close() self.fail('%r is an invalid file mode' % mode) + def testModeStrings(self): + # test that the mode attribute is correct for various mode strings + # given as init args + try: + for modes in [('w', 'wb'), ('wb', 'wb'), ('wb+', 'rb+'), + ('w+b', 'rb+'), ('a', 'ab'), ('ab', 'ab'), + ('ab+', 'ab+'), ('a+b', 'ab+'), ('r', 'rb'), + ('rb', 'rb'), ('rb+', 'rb+'), ('r+b', 'rb+')]: + # read modes are last so that TESTFN will exist first + with _FileIO(TESTFN, modes[0]) as f: + self.assertEqual(f.mode, modes[1]) + finally: + if os.path.exists(TESTFN): + os.unlink(TESTFN) + def testUnicodeOpen(self): # verify repr works for unicode too f = _FileIO(str(TESTFN), "w") diff --git a/Misc/ACKS b/Misc/ACKS index 8977c02fa28..8d29290a709 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -124,6 +124,7 @@ Monty Brandenberg Georg Brandl Christopher Brannon Terrence Brannon +Erik Bray Brian Brazil Dave Brennan Tom Bridgman diff --git a/Misc/NEWS b/Misc/NEWS index afb7464be0f..7ef400ea888 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #18876: The FileIO.mode attribute now better reflects the actual mode + under which the file was opened. Patch by Erik Bray. + - Issue #18851: Avoid a double close of subprocess pipes when the child process fails starting. diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 6cd7d817c8d..5946e6a2bcc 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -47,6 +47,7 @@ typedef struct { int fd; unsigned int readable : 1; unsigned int writable : 1; + unsigned int appending : 1; signed int seekable : 2; /* -1 means unknown */ unsigned int closefd : 1; PyObject *weakreflist; @@ -124,6 +125,7 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->fd = -1; self->readable = 0; self->writable = 0; + self->appending = 0; self->seekable = -1; self->closefd = 1; self->weakreflist = NULL; @@ -184,7 +186,7 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) Py_UNICODE *widename = NULL; #endif int ret = 0; - int rwa = 0, plus = 0, append = 0; + int rwa = 0, plus = 0; int flags = 0; int fd = -1; int closefd = 1; @@ -279,8 +281,8 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) goto bad_mode; rwa = 1; self->writable = 1; - flags |= O_CREAT; - append = 1; + self->appending = 1; + flags |= O_APPEND | O_CREAT; break; case 'b': break; @@ -311,11 +313,6 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) flags |= O_BINARY; #endif -#ifdef O_APPEND - if (append) - flags |= O_APPEND; -#endif - if (fd >= 0) { if (check_fd(fd)) goto error; @@ -356,7 +353,7 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) if (PyObject_SetAttrString((PyObject *)self, "name", nameobj) < 0) goto error; - if (append) { + if (self->appending) { /* For consistent behaviour, we explicitly seek to the end of file (otherwise, it might be done only on the first write()). */ @@ -898,7 +895,13 @@ fileio_truncate(fileio *self, PyObject *args) static char * mode_string(fileio *self) { - if (self->readable) { + if (self->appending) { + if (self->readable) + return "ab+"; + else + return "ab"; + } + else if (self->readable) { if (self->writable) return "rb+"; else From bef29fb3fa3118c63dfd8ba4baa13f2fed055fba Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Sep 2013 00:26:15 +0200 Subject: [PATCH 346/480] Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast 64-bit pointer to long (32 bits). --- Misc/NEWS | 3 +++ Modules/_tkinter.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 7ef400ea888..14201ca4869 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast + 64-bit pointer to long (32 bits). + - Issue #18876: The FileIO.mode attribute now better reflects the actual mode under which the file was opened. Patch by Erik Bray. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 52ee881373f..0627d60b677 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -2814,7 +2814,7 @@ Tkapp_InterpAddr(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, ":interpaddr")) return NULL; - return PyInt_FromLong((long)Tkapp_Interp(self)); + return PyLong_FromVoidPtr(Tkapp_Interp(self)); } static PyObject * From 6169b814fd0bd4bb635c20432639a1c04cfe45eb Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 5 Sep 2013 16:04:35 +0200 Subject: [PATCH 347/480] Issue #18709: GCC 4.6 complains that 'v' may be used uninitialized in GEN_EMAIL/GEN_URI/GEN_DNS case --- Modules/_ssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index a045be421c4..a0aebd03985 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -670,7 +670,7 @@ _get_peer_alt_names (X509 *certificate) { int i, j; PyObject *peer_alt_names = Py_None; - PyObject *v, *t; + PyObject *v = NULL, *t; X509_EXTENSION *ext = NULL; GENERAL_NAMES *names = NULL; GENERAL_NAME *name; From 221488b2a4346205a458795e633c4a600712ab6b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 5 Sep 2013 17:28:10 +0300 Subject: [PATCH 348/480] Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. --- Lib/inspect.py | 3 ++- Lib/test/inspect_fodder.py | 2 ++ Lib/test/test_inspect.py | 19 +++++++++++++++++-- Misc/NEWS | 3 +++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index c2a32fee246..933694394dd 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -728,7 +728,8 @@ def getclasstree(classes, unique=0): for parent in c.__bases__: if not parent in children: children[parent] = [] - children[parent].append(c) + if c not in children[parent]: + children[parent].append(c) if unique and parent in classes: break elif c not in roots: roots.append(c) diff --git a/Lib/test/inspect_fodder.py b/Lib/test/inspect_fodder.py index afde2e2514b..5c87ae6f828 100644 --- a/Lib/test/inspect_fodder.py +++ b/Lib/test/inspect_fodder.py @@ -49,6 +49,8 @@ def argue(self, a, b, c): class MalodorousPervert(StupidGit): pass +Tit = MalodorousPervert + class ParrotDroppings: pass diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 04dcfe992a4..4130cd0b502 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -220,8 +220,23 @@ def test_getclasses(self): [('FesteringGob', mod.FesteringGob), ('MalodorousPervert', mod.MalodorousPervert), ('ParrotDroppings', mod.ParrotDroppings), - ('StupidGit', mod.StupidGit)]) - tree = inspect.getclasstree([cls[1] for cls in classes], 1) + ('StupidGit', mod.StupidGit), + ('Tit', mod.MalodorousPervert), + ]) + tree = inspect.getclasstree([cls[1] for cls in classes]) + self.assertEqual(tree, + [(mod.ParrotDroppings, ()), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ], + (mod.StupidGit, ()), + [(mod.MalodorousPervert, (mod.StupidGit,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ] + ] + ]) + tree = inspect.getclasstree([cls[1] for cls in classes], True) self.assertEqual(tree, [(mod.ParrotDroppings, ()), (mod.StupidGit, ()), diff --git a/Misc/NEWS b/Misc/NEWS index 14201ca4869..3bbced36d0f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #18830: inspect.getclasstree() no more produces duplicated entries even + when input list contains duplicates. + - Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast 64-bit pointer to long (32 bits). From 515507478e60338e68264b23caeefb09a383ffd9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 5 Sep 2013 18:02:57 +0300 Subject: [PATCH 349/480] Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in the _sre moduel. --- Misc/NEWS | 3 +++ Modules/_sre.c | 36 ++++++++++++++++++++++-------------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 3bbced36d0f..8c67ad1d3a1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in + the _sre moduel. + - Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. diff --git a/Modules/_sre.c b/Modules/_sre.c index c9e4f7355e9..a7103cc456b 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -272,7 +272,7 @@ data_stack_grow(SRE_STATE* state, Py_ssize_t size) if (cursize < minsize) { void* stack; cursize = minsize+minsize/4+1024; - TRACE(("allocate/grow stack %d\n", cursize)); + TRACE(("allocate/grow stack %" PY_FORMAT_SIZE_T "d\n", cursize)); stack = PyMem_REALLOC(state->data_stack, cursize); if (!stack) { data_stack_dealloc(state); @@ -592,12 +592,13 @@ SRE_COUNT(SRE_STATE* state, SRE_CODE* pattern, Py_ssize_t maxcount) if (!i) break; } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, + TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr, (SRE_CHAR*) state->ptr - ptr)); return (SRE_CHAR*) state->ptr - ptr; } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, ptr - (SRE_CHAR*) state->ptr)); + TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr, + ptr - (SRE_CHAR*) state->ptr)); return ptr - (SRE_CHAR*) state->ptr; } @@ -684,7 +685,8 @@ SRE_INFO(SRE_STATE* state, SRE_CODE* pattern) #define DATA_STACK_ALLOC(state, type, ptr) \ do { \ alloc_pos = state->data_stack_base; \ - TRACE(("allocating %s in %d (%d)\n", \ + TRACE(("allocating %s in %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ SFY(type), alloc_pos, sizeof(type))); \ if (sizeof(type) > state->data_stack_size - alloc_pos) { \ int j = data_stack_grow(state, sizeof(type)); \ @@ -698,13 +700,14 @@ do { \ #define DATA_STACK_LOOKUP_AT(state, type, ptr, pos) \ do { \ - TRACE(("looking up %s at %d\n", SFY(type), pos)); \ + TRACE(("looking up %s at %" PY_FORMAT_SIZE_T "d\n", SFY(type), pos)); \ ptr = (type*)(state->data_stack+pos); \ } while (0) #define DATA_STACK_PUSH(state, data, size) \ do { \ - TRACE(("copy data in %p to %d (%d)\n", \ + TRACE(("copy data in %p to %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ data, state->data_stack_base, size)); \ if (size > state->data_stack_size - state->data_stack_base) { \ int j = data_stack_grow(state, size); \ @@ -718,7 +721,8 @@ do { \ #define DATA_STACK_POP(state, data, size, discard) \ do { \ - TRACE(("copy data to %p from %d (%d)\n", \ + TRACE(("copy data to %p from %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ data, state->data_stack_base-size, size)); \ memcpy(data, state->data_stack+state->data_stack_base-size, size); \ if (discard) \ @@ -727,7 +731,8 @@ do { \ #define DATA_STACK_POP_DISCARD(state, size) \ do { \ - TRACE(("discard data from %d (%d)\n", \ + TRACE(("discard data from %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ state->data_stack_base-size, size)); \ state->data_stack_base -= size; \ } while(0) @@ -831,8 +836,9 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) /* optimization info block */ /* <1=skip> <2=flags> <3=min> ... */ if (ctx->pattern[3] && (end - ctx->ptr) < ctx->pattern[3]) { - TRACE(("reject (got %d chars, need %d)\n", - (end - ctx->ptr), ctx->pattern[3])); + TRACE(("reject (got %" PY_FORMAT_SIZE_T "d chars, " + "need %" PY_FORMAT_SIZE_T "d)\n", + (end - ctx->ptr), (Py_ssize_t) ctx->pattern[3])); RETURN_FAILURE; } ctx->pattern += ctx->pattern[1] + 1; @@ -1207,7 +1213,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) ctx->count = ctx->u.rep->count+1; - TRACE(("|%p|%p|MAX_UNTIL %d\n", ctx->pattern, + TRACE(("|%p|%p|MAX_UNTIL %" PY_FORMAT_SIZE_T "d\n", ctx->pattern, ctx->ptr, ctx->count)); if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { @@ -1270,7 +1276,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) ctx->count = ctx->u.rep->count+1; - TRACE(("|%p|%p|MIN_UNTIL %d %p\n", ctx->pattern, + TRACE(("|%p|%p|MIN_UNTIL %" PY_FORMAT_SIZE_T "d %p\n", ctx->pattern, ctx->ptr, ctx->count, ctx->u.rep->pattern)); if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { @@ -1483,7 +1489,8 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) TRACE(("|%p|%p|JUMP_ASSERT_NOT\n", ctx->pattern, ctx->ptr)); goto jump_assert_not; case JUMP_NONE: - TRACE(("|%p|%p|RETURN %d\n", ctx->pattern, ctx->ptr, ret)); + TRACE(("|%p|%p|RETURN %" PY_FORMAT_SIZE_T "d\n", ctx->pattern, + ctx->ptr, ret)); break; } @@ -1532,7 +1539,8 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) pattern += 1 + pattern[1]; } - TRACE(("prefix = %p %d %d\n", prefix, prefix_len, prefix_skip)); + TRACE(("prefix = %p %" PY_FORMAT_SIZE_T "d %" PY_FORMAT_SIZE_T "d\n", + prefix, prefix_len, prefix_skip)); TRACE(("charset = %p\n", charset)); #if defined(USE_FAST_SEARCH) From 4a272bc45eeda659d1f7f1b733ec5d9fe90497e3 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 5 Sep 2013 22:57:20 +0100 Subject: [PATCH 350/480] Issue #18933: Added links to source code. --- Doc/library/logging.config.rst | 4 ++++ Doc/library/logging.handlers.rst | 4 ++++ Doc/library/logging.rst | 3 +++ 3 files changed, 11 insertions(+) diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index ee141f1a4b9..2dc82acf3f6 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -17,6 +17,10 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/config.py` + +-------------- + This section describes the API for configuring the logging module. .. _logging-config-api: diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 534c35c0e10..3c05e9ae13d 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -17,6 +17,10 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/handlers.py` + +-------------- + .. currentmodule:: logging The following useful handlers are provided in the package. Note that three of diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index a2d903365bf..d3984c78440 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -20,6 +20,9 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/__init__.py` + +-------------- .. versionadded:: 2.3 From d38360eea56e307d45affe7dd4f051d42067ee18 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Fri, 6 Sep 2013 10:09:45 +0100 Subject: [PATCH 351/480] Issue #18940: Handled low-volume logging when delay is True. --- Lib/logging/handlers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index c4b8746ee5a..8a813d3e375 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -137,7 +137,9 @@ def doRollover(self): dfn = self.baseFilename + ".1" if os.path.exists(dfn): os.remove(dfn) - os.rename(self.baseFilename, dfn) + # Issue 18940: A file may not have been created if delay is True. + if os.path.exists(self.baseFilename): + os.rename(self.baseFilename, dfn) #print "%s -> %s" % (self.baseFilename, dfn) self.stream = self._open() @@ -343,7 +345,9 @@ def doRollover(self): dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple) if os.path.exists(dfn): os.remove(dfn) - os.rename(self.baseFilename, dfn) + # Issue 18940: A file may not have been created if delay is True. + if os.path.exists(self.baseFilename): + os.rename(self.baseFilename, dfn) if self.backupCount > 0: # find the oldest log file and delete it #s = glob.glob(self.baseFilename + ".20*") From 304d5a3f2dbcf84601f5c350a3bfc0d3db063e58 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Fri, 6 Sep 2013 10:24:08 +0100 Subject: [PATCH 352/480] Issue #18941: Respected delay when doing rollover. --- Lib/logging/__init__.py | 1 + Lib/logging/handlers.py | 13 ++++--------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 1a622a3d074..0e8e3ddde0d 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -893,6 +893,7 @@ def __init__(self, filename, mode='a', encoding=None, delay=0): self.baseFilename = os.path.abspath(filename) self.mode = mode self.encoding = encoding + self.delay = delay if delay: #We don't open the stream, but we still need to call the #Handler constructor to set level, formatter, lock etc. diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 8a813d3e375..c45a3132ba7 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -140,8 +140,8 @@ def doRollover(self): # Issue 18940: A file may not have been created if delay is True. if os.path.exists(self.baseFilename): os.rename(self.baseFilename, dfn) - #print "%s -> %s" % (self.baseFilename, dfn) - self.stream = self._open() + if not self.delay: + self.stream = self._open() def shouldRollover(self, record): """ @@ -349,15 +349,10 @@ def doRollover(self): if os.path.exists(self.baseFilename): os.rename(self.baseFilename, dfn) if self.backupCount > 0: - # find the oldest log file and delete it - #s = glob.glob(self.baseFilename + ".20*") - #if len(s) > self.backupCount: - # s.sort() - # os.remove(s[0]) for s in self.getFilesToDelete(): os.remove(s) - #print "%s -> %s" % (self.baseFilename, dfn) - self.stream = self._open() + if not self.delay: + self.stream = self._open() newRolloverAt = self.computeRollover(currentTime) while newRolloverAt <= currentTime: newRolloverAt = newRolloverAt + self.interval From c78836d93c72b2d8c1b3f2ce055dc4b700f523f1 Mon Sep 17 00:00:00 2001 From: Eli Bendersky Date: Fri, 6 Sep 2013 06:17:15 -0700 Subject: [PATCH 353/480] Close #18849: Fixed a Windows-specific tempfile bug where collision with an existing directory caused mkstemp and related APIs to fail instead of retrying. Report and fix by Vlad Shcherbina. --- Lib/tempfile.py | 4 ++++ Lib/test/test_tempfile.py | 26 ++++++++++++++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 5 +++++ 4 files changed, 36 insertions(+) diff --git a/Lib/tempfile.py b/Lib/tempfile.py index b5dce974e18..7154d2c6b52 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -242,6 +242,10 @@ def _mkstemp_inner(dir, pre, suf, flags): except OSError, e: if e.errno == _errno.EEXIST: continue # try again + if _os.name == 'nt' and e.errno == _errno.EACCES: + # On windows, when a directory with the chosen name already + # exists, EACCES error code is returned instead of EEXIST. + continue raise raise IOError, (_errno.EEXIST, "No usable temporary file name found") diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index b4d23addf2b..eb3e2b9a6dd 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -386,6 +386,32 @@ def test_textmode(self): self.do_create(bin=0).write("blat\n") # XXX should test that the file really is a text file + def test_collision_with_existing_directory(self): + # _mkstemp_inner tries another name when a directory with + # the chosen name already exists + container_dir = tempfile.mkdtemp() + try: + def mock_get_candidate_names(): + return iter(['aaa', 'aaa', 'bbb']) + with support.swap_attr(tempfile, + '_get_candidate_names', + mock_get_candidate_names): + dir = tempfile.mkdtemp(dir=container_dir) + self.assertTrue(dir.endswith('aaa')) + + flags = tempfile._bin_openflags + (fd, name) = tempfile._mkstemp_inner(container_dir, + tempfile.template, + '', + flags) + try: + self.assertTrue(name.endswith('bbb')) + finally: + os.close(fd) + os.unlink(name) + finally: + support.rmtree(container_dir) + test_classes.append(test__mkstemp_inner) diff --git a/Misc/ACKS b/Misc/ACKS index 8d29290a709..e251596c988 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -936,6 +936,7 @@ Aman Shah Ha Shao Mark Shannon Richard Shapiro +Vlad Shcherbina Justin Sheehy Charlie Shepherd Bruce Sherwood diff --git a/Misc/NEWS b/Misc/NEWS index 8c67ad1d3a1..53be4c14ffb 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -156,6 +156,11 @@ Library - Issue #18113: Fixed a refcount leak in the curses.panel module's set_userptr() method. Reported by Atsuo Ishimoto. +- Issue #18849: Fixed a Windows-specific tempfile bug where collision with an + existing directory caused mkstemp and related APIs to fail instead of + retrying. Report and fix by Vlad Shcherbina. + + Tools/Demos ----------- From 78da53968e81639cdbb34e2fb01c399e1c0dba03 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 7 Sep 2013 15:19:30 +0300 Subject: [PATCH 354/480] #18894: remove mention of deprecated fail* methods. --- Doc/library/unittest.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index f36a44deccf..5e6ffb8ef26 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1603,8 +1603,7 @@ Loading and running tests A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.fail\*` or - :meth:`TestCase.assert\*` methods. + was explicitly signalled using the :meth:`TestCase.assert\*` methods. .. versionchanged:: 2.2 Contains formatted tracebacks instead of :func:`sys.exc_info` results. From 2b8ab86edd73ddb6fa141fde8f7641da64ef53d7 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 7 Sep 2013 15:23:36 +0300 Subject: [PATCH 355/480] #18895: split a sentence in unittest docs. --- Doc/library/unittest.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 5e6ffb8ef26..30df44e7fb7 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1703,7 +1703,7 @@ Loading and running tests .. method:: addError(test, err) - Called when the test case *test* raises an unexpected exception *err* is a + Called when the test case *test* raises an unexpected exception. *err* is a tuple of the form returned by :func:`sys.exc_info`: ``(type, value, traceback)``. From 3b8f57506769d83fc11b3bf557bf076bec33e109 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 7 Sep 2013 11:27:45 -0700 Subject: [PATCH 356/480] os.isatty is not Unix only. Correct the wrong documentation. Addresses issue #18553 --- Doc/library/os.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 106c1df35c0..d4912ce9c34 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -737,8 +737,6 @@ as internal buffering of data. Return ``True`` if the file descriptor *fd* is open and connected to a tty(-like) device, else ``False``. - Availability: Unix. - .. function:: lseek(fd, pos, how) From 77e0024b68d6eec8b9c95cddd39f7050439e9938 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 7 Sep 2013 17:50:35 -0700 Subject: [PATCH 357/480] Correct Profile class usage example. Addresses issue #18033 . Patch contributed by Olivier Hervieu and Dmi Baranov. --- Doc/library/profile.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index e2757f6632a..00abef2873e 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -267,14 +267,16 @@ functions: Directly using the :class:`Profile` class allows formatting profile results without writing the profile data to a file:: - import cProfile, pstats, io + import cProfile, pstats, StringIO pr = cProfile.Profile() pr.enable() - ... do something ... + # ... do something ... pr.disable() - s = io.StringIO() - ps = pstats.Stats(pr, stream=s) - ps.print_results() + s = StringIO.StringIO() + sortby = 'cumulative' + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats() + print s.getvalue() .. method:: enable() From de313450816e2a492595114276f8980818e4b883 Mon Sep 17 00:00:00 2001 From: Michael Foord Date: Sun, 8 Sep 2013 15:34:27 +1200 Subject: [PATCH 358/480] Closes issue 14971. unittest test discovery no longer gets confused when a function has a different __name__ than its name in the TestCase class dictionary. --- Lib/unittest/loader.py | 4 +++- Lib/unittest/test/test_loader.py | 15 +++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index e88f536b394..c6fc6630de1 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -106,7 +106,9 @@ def loadTestsFromName(self, name, module=None): elif (isinstance(obj, types.UnboundMethodType) and isinstance(parent, type) and issubclass(parent, case.TestCase)): - return self.suiteClass([parent(obj.__name__)]) + name = parts[-1] + inst = parent(name) + return self.suiteClass([inst]) elif isinstance(obj, suite.TestSuite): return obj elif hasattr(obj, '__call__'): diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py index d8d52f7c0df..3e9756e65ae 100644 --- a/Lib/unittest/test/test_loader.py +++ b/Lib/unittest/test/test_loader.py @@ -1281,6 +1281,21 @@ def test_suiteClass__default_value(self): loader = unittest.TestLoader() self.assertTrue(loader.suiteClass is unittest.TestSuite) + # Make sure the dotted name resolution works even if the actual + # function doesn't have the same name as is used to find it. + def test_loadTestsFromName__function_with_different_name_than_method(self): + # lambdas have the name ''. + m = types.ModuleType('m') + class MyTestCase(unittest.TestCase): + test = lambda: 1 + m.testcase_1 = MyTestCase + + loader = unittest.TestLoader() + suite = loader.loadTestsFromNames(['testcase_1.test'], m) + self.assertIsInstance(suite, loader.suiteClass) + + ref_suite = unittest.TestSuite([MyTestCase('test')]) + self.assertEqual(list(suite), [ref_suite]) if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 53be4c14ffb..3956628edb3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #14971: unittest test discovery no longer gets confused when a function + has a different __name__ than its name in the TestCase class dictionary. + - Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in the _sre moduel. From 7fd9d17685d68ded651f13c0377218fa5372c130 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 7 Sep 2013 23:18:53 -0700 Subject: [PATCH 359/480] Fix the dead link of IEEE_854-1987 standard with the Wikipedia entry. Addresses issue #18438 --- Lib/decimal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/decimal.py b/Lib/decimal.py index 361126f6d0e..04bf5c26605 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -25,7 +25,7 @@ and IEEE standard 854-1987: - www.cs.berkeley.edu/~ejr/projects/754/private/drafts/854-1987/dir.html + http://en.wikipedia.org/wiki/IEEE_854-1987 Decimal floating point has finite precision with arbitrarily large bounds. From 37b9cd5d02d4091a60ed289915fd043d7535d409 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 8 Sep 2013 20:32:56 +0300 Subject: [PATCH 360/480] Fixed tests with Tcl/Tk <8.5 (closes #18964). --- Lib/test/test_tcl.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 1e60b1d1a8d..50e1ac40acf 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -13,6 +13,14 @@ from Tkinter import Tcl from _tkinter import TclError +tcl_version = _tkinter.TCL_VERSION.split('.') +try: + for i in range(len(tcl_version)): + tcl_version[i] = int(tcl_version[i]) +except ValueError: + pass +tcl_version = tuple(tcl_version) + class TkinterTest(unittest.TestCase): @@ -209,9 +217,12 @@ def test_splitlist(self): (('a', 3.4), ('a', 3.4)), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), - (call('dict', 'create', 1, u'\u20ac', '\xe2\x82\xac', (3.4,)), - (1, u'\u20ac', u'\u20ac', (3.4,))), ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 1, u'\u20ac', '\xe2\x82\xac', (3.4,)), + (1, u'\u20ac', u'\u20ac', (3.4,))), + ] for arg, res in testcases: self.assertEqual(splitlist(arg), res) self.assertRaises(TclError, splitlist, '{') @@ -243,9 +254,12 @@ def test_split(self): (('a', (2, 3.4)), ('a', (2, 3.4))), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), - (call('dict', 'create', 12, u'\u20ac', '\xe2\x82\xac', (3.4,)), - (12, u'\u20ac', u'\u20ac', (3.4,))), ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 12, u'\u20ac', '\xe2\x82\xac', (3.4,)), + (12, u'\u20ac', u'\u20ac', (3.4,))), + ] for arg, res in testcases: self.assertEqual(split(arg), res) From 47e4df31958cbad2f8f4dced6ea5ab9a31845b4e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 8 Sep 2013 20:43:16 +0300 Subject: [PATCH 361/480] Fix a typo. (closes #18953) --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 3956628edb3..d239b349039 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,7 +36,7 @@ Library has a different __name__ than its name in the TestCase class dictionary. - Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in - the _sre moduel. + the _sre module. - Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. From 14fe08094fac1b6f8260140c6445150485da7e7f Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Mon, 9 Sep 2013 10:32:08 +0100 Subject: [PATCH 362/480] Docstring typo fix: Arithmentic -> Arithmetic. --- Lib/unittest/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py index 201a3f0d2da..a5d50af78f2 100644 --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -11,7 +11,7 @@ import unittest - class IntegerArithmenticTestCase(unittest.TestCase): + class IntegerArithmeticTestCase(unittest.TestCase): def testAdd(self): ## test method names begin 'test*' self.assertEqual((1 + 2), 3) self.assertEqual(0 + 1, 1) From d3d0e237a4a8dea3e1ea21210a4857f97c673891 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Mon, 9 Sep 2013 22:38:58 -0700 Subject: [PATCH 363/480] Clarify mmap.close method behavior. Addresses issue #18815 Patch contributed by Anoop Thomas Mathew. --- Doc/library/mmap.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 58d00c81cae..ac1963f5bbe 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -152,8 +152,9 @@ memory but does not update the underlying file. .. method:: close() - Close the file. Subsequent calls to other methods of the object will - result in an exception being raised. + Closes the mmap. Subsequent calls to other methods of the object will + result in a ValueError exception being raised. This will not close + the open file. .. method:: find(string[, start[, end]]) From 1ca551f7c93e98b26e9d52446eb7de2c6ca6fd4a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 11 Sep 2013 22:49:06 +0300 Subject: [PATCH 364/480] Issue #18988: The "Tab" key now works when a word is already autocompleted. --- Lib/idlelib/AutoComplete.py | 9 +++------ Lib/idlelib/AutoCompleteWindow.py | 3 ++- Misc/NEWS | 2 ++ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/idlelib/AutoComplete.py b/Lib/idlelib/AutoComplete.py index 4e173252d61..cb38691097e 100644 --- a/Lib/idlelib/AutoComplete.py +++ b/Lib/idlelib/AutoComplete.py @@ -156,12 +156,9 @@ def open_completions(self, evalfuncs, complete, userWantsWin, mode=None): if not comp_lists[0]: return self.autocompletewindow = self._make_autocomplete_window() - self.autocompletewindow.show_window(comp_lists, - "insert-%dc" % len(comp_start), - complete, - mode, - userWantsWin) - return True + return not self.autocompletewindow.show_window( + comp_lists, "insert-%dc" % len(comp_start), + complete, mode, userWantsWin) def fetch_completions(self, what, mode): """Return a pair of lists of completions for something. The first list diff --git a/Lib/idlelib/AutoCompleteWindow.py b/Lib/idlelib/AutoCompleteWindow.py index 298177fc847..27b0e56bd58 100644 --- a/Lib/idlelib/AutoCompleteWindow.py +++ b/Lib/idlelib/AutoCompleteWindow.py @@ -157,13 +157,14 @@ def show_window(self, comp_lists, index, complete, mode, userWantsWin): self.start = self.widget.get(self.startindex, "insert") if complete: completed = self._complete_string(self.start) + start = self.start self._change_start(completed) i = self._binary_search(completed) if self.completions[i] == completed and \ (i == len(self.completions)-1 or self.completions[i+1][:len(completed)] != completed): # There is exactly one matching completion - return + return completed == start self.userwantswindow = userWantsWin self.lasttypedstart = self.start diff --git a/Misc/NEWS b/Misc/NEWS index d239b349039..9c0b13fb029 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -191,6 +191,8 @@ Build IDLE ---- +- Issue #18988: The "Tab" key now works when a word is already autocompleted. + - Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster. - Issue #18429: Format / Format Paragraph, now works when comment blocks From db02316b2e5138e792d5036b52db7c91f64142f1 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 11 Sep 2013 22:52:58 -0700 Subject: [PATCH 365/480] Improve the docstring of random.shuffle. Inform users not to provide int arg. Addresses issue #14927 --- Lib/random.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/random.py b/Lib/random.py index af04ab207d3..6fa1c050889 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -278,6 +278,8 @@ def shuffle(self, x, random=None, int=int): Optional arg random is a 0-argument function returning a random float in [0.0, 1.0); by default, the standard random.random. + + Do not supply the 'int' argument. """ if random is None: From fbcf25247f3822be7b25da0c64727675d155faa9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 13 Sep 2013 07:52:00 +0300 Subject: [PATCH 366/480] Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, if all necessary functions are already found in libuuid. Patch by Evgeny Sologubov. --- Lib/uuid.py | 2 ++ Misc/ACKS | 1 + Misc/NEWS | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/Lib/uuid.py b/Lib/uuid.py index fdd0c5cbec0..cb4e5f050f1 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -408,6 +408,8 @@ def _netbios_getnode(): _uuid_generate_random = lib.uuid_generate_random if hasattr(lib, 'uuid_generate_time'): _uuid_generate_time = lib.uuid_generate_time + if _uuid_generate_random is not None: + break # found everything we were looking for # The uuid_generate_* functions are broken on MacOS X 10.5, as noted # in issue #8621 the function generates the same sequence of values diff --git a/Misc/ACKS b/Misc/ACKS index e251596c988..6f2b3a76fd9 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -970,6 +970,7 @@ Roy Smith Rafal Smotrzyk Dirk Soede Paul Sokolovsky +Evgeny Sologubov Cody Somerville Clay Spence Stefan Sperling diff --git a/Misc/NEWS b/Misc/NEWS index 9c0b13fb029..ea74d71bd9a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,10 @@ Core and Builtins Library ------- +- Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, + if all necessary functions are already found in libuuid. + Patch by Evgeny Sologubov. + - Issue #14971: unittest test discovery no longer gets confused when a function has a different __name__ than its name in the TestCase class dictionary. From 6eeca0505a22fd5e6f9fdaf24168b7fd490ade23 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Fri, 13 Sep 2013 00:18:55 -0700 Subject: [PATCH 367/480] Fix SimpleHTTPServer's request handling case on trailing '/'. Patch contributed by Vajrasky Kok. Addresses Issue #17324 --- Lib/SimpleHTTPServer.py | 4 ++++ Lib/test/test_httpservers.py | 3 +++ Misc/NEWS | 3 +++ 3 files changed, 10 insertions(+) diff --git a/Lib/SimpleHTTPServer.py b/Lib/SimpleHTTPServer.py index 3e0334da3b3..1996f4d29ba 100644 --- a/Lib/SimpleHTTPServer.py +++ b/Lib/SimpleHTTPServer.py @@ -149,6 +149,8 @@ def translate_path(self, path): # abandon query parameters path = path.split('?',1)[0] path = path.split('#',1)[0] + # Don't forget explicit trailing slash when normalizing. Issue17324 + trailing_slash = True if path.rstrip().endswith('/') else False path = posixpath.normpath(urllib.unquote(path)) words = path.split('/') words = filter(None, words) @@ -158,6 +160,8 @@ def translate_path(self, path): head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) + if trailing_slash: + path += '/' return path def copyfile(self, source, outputfile): diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 5dcedc0fe8b..3f1436004d5 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -313,6 +313,9 @@ def test_get(self): #constructs the path relative to the root directory of the HTTPServer response = self.request(self.tempdir_name + '/test') self.check_status_and_reason(response, 200, data=self.data) + # check for trailing "/" which should return 404. See Issue17324 + response = self.request(self.tempdir_name + '/test/') + self.check_status_and_reason(response, 404) response = self.request(self.tempdir_name + '/') self.check_status_and_reason(response, 200) response = self.request(self.tempdir_name) diff --git a/Misc/NEWS b/Misc/NEWS index ea74d71bd9a..e025df630d6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #17324: Fix http.server's request handling case on trailing '/'. Patch + contributed by Vajrasky Kok. + - Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, if all necessary functions are already found in libuuid. Patch by Evgeny Sologubov. From 7bc4c8c21e8b7ee50c21e1dd38564f3b5cfadabb Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Fri, 13 Sep 2013 22:17:40 +0300 Subject: [PATCH 368/480] #18951: use consistent names in unittest docs. --- Doc/library/unittest.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 30df44e7fb7..e5faa9a50f2 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -916,8 +916,8 @@ Test cases | :meth:`assertRaises(exc, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | | | ` | | | +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertRaisesRegexp(exc, re, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 2.7 | - | ` | and the message matches *re* | | + | :meth:`assertRaisesRegexp(exc, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 2.7 | + | ` | and the message matches regex *r* | | +---------------------------------------------------------+--------------------------------------+------------+ .. method:: assertRaises(exception, callable, *args, **kwds) @@ -993,10 +993,10 @@ Test cases | :meth:`assertLessEqual(a, b) | ``a <= b`` | 2.7 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertRegexpMatches(s, re) | ``regex.search(s)`` | 2.7 | + | :meth:`assertRegexpMatches(s, r) | ``r.search(s)`` | 2.7 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotRegexpMatches(s, re) | ``not regex.search(s)`` | 2.7 | + | :meth:`assertNotRegexpMatches(s, r) | ``not r.search(s)`` | 2.7 | | ` | | | +---------------------------------------+--------------------------------+--------------+ | :meth:`assertItemsEqual(a, b) | sorted(a) == sorted(b) and | 2.7 | From f2e8b678e4782b393c8a6e59f2ecda9afdd64a12 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Fri, 13 Sep 2013 22:55:08 +0300 Subject: [PATCH 369/480] Fix markup in email.parser docs. --- Doc/library/email.parser.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index 6cc0494cde5..0f99a2fb040 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -168,7 +168,7 @@ in the top-level :mod:`email` package namespace. Return a message object structure from a string. This is exactly equivalent to ``Parser().parsestr(s)``. Optional *_class* and *strict* are interpreted as - with the :class:``~email.parser.Parser` class constructor. + with the :class:`~email.parser.Parser` class constructor. .. versionchanged:: 2.2.2 The *strict* flag was added. @@ -178,7 +178,7 @@ in the top-level :mod:`email` package namespace. Return a message object structure tree from an open file object. This is exactly equivalent to ``Parser().parse(fp)``. Optional *_class* and *strict* - are interpreted as with the :class:``~email.parser.Parser` class constructor. + are interpreted as with the :class:`~email.parser.Parser` class constructor. .. versionchanged:: 2.2.2 The *strict* flag was added. From d64cc2a8fafd721ea05c9f71d677453872d990b9 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Fri, 13 Sep 2013 23:52:12 +0300 Subject: [PATCH 370/480] #18981: fix a typo in a comment (noticed by Anoop Thomas Mathew). --- Lib/ctypes/test/test_integers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ctypes/test/test_integers.py b/Lib/ctypes/test/test_integers.py index 5b6453a002a..62e4b08a2b6 100644 --- a/Lib/ctypes/test/test_integers.py +++ b/Lib/ctypes/test/test_integers.py @@ -1,4 +1,4 @@ -# superseeded by test_numbers.py +# superseded by test_numbers.py import unittest if __name__ == '__main__': From 7f6fde4657f73e84054fba69f4dc1f64f1e92c14 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 14 Sep 2013 22:17:39 -0700 Subject: [PATCH 371/480] Issue #19018: The heapq.merge() function no longer suppresses IndexError --- Lib/heapq.py | 15 ++++++++++----- Lib/test/test_heapq.py | 9 +++++++++ Misc/ACKS | 2 ++ Misc/NEWS | 3 +++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/Lib/heapq.py b/Lib/heapq.py index ca79db15295..19037ab1290 100644 --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -366,6 +366,7 @@ def merge(*iterables): ''' _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration + _len = len h = [] h_append = h.append @@ -377,17 +378,21 @@ def merge(*iterables): pass heapify(h) - while 1: + while _len(h) > 1: try: - while 1: - v, itnum, next = s = h[0] # raises IndexError when h is empty + while True: + v, itnum, next = s = h[0] yield v s[0] = next() # raises StopIteration when exhausted _heapreplace(h, s) # restore heap condition except _StopIteration: _heappop(h) # remove empty iterator - except IndexError: - return + if h: + # fast case when only a single iterator remains + v, itnum, next = h[0] + yield v + for v in next.__self__: + yield v # Extend the implementations of nsmallest and nlargest to use a key= argument _nsmallest = nsmallest diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py index 73b88f09b9b..c4de593bb82 100644 --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -158,6 +158,15 @@ def test_merge(self): self.assertEqual(sorted(chain(*inputs)), list(self.module.merge(*inputs))) self.assertEqual(list(self.module.merge()), []) + def test_merge_does_not_suppress_index_error(self): + # Issue 19018: Heapq.merge suppresses IndexError from user generator + def iterable(): + s = list(range(10)) + for i in range(20): + yield s[i] # IndexError when i > 10 + with self.assertRaises(IndexError): + list(self.module.merge(iterable(), iterable())) + def test_merge_stability(self): class Int(int): pass diff --git a/Misc/ACKS b/Misc/ACKS index 6f2b3a76fd9..6048ed8dd71 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -110,6 +110,7 @@ Paul Boddie Matthew Boedicker Robin Boerdijk David Bolen +Wouter Bolsterlee Gawain Bolton Gregory Bond Jurjen Bos @@ -313,6 +314,7 @@ Nils Fischbeck Frederik Fix Matt Fleming Hernán Martínez Foffani +Artem Fokin Arnaud Fontaine Michael Foord Amaury Forgeot d'Arc diff --git a/Misc/NEWS b/Misc/NEWS index e025df630d6..c4698f4d7ff 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -35,6 +35,9 @@ Library - Issue #17324: Fix http.server's request handling case on trailing '/'. Patch contributed by Vajrasky Kok. +- Issue #19018: The heapq.merge() function no longer suppresses IndexError + in the underlying iterables. + - Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, if all necessary functions are already found in libuuid. Patch by Evgeny Sologubov. From aa5e726ab1bf1a45833c6ed8f135d5d3681b2c0a Mon Sep 17 00:00:00 2001 From: Andrew Kuchling Date: Sun, 15 Sep 2013 13:11:47 -0400 Subject: [PATCH 372/480] #16042: CVE-2013-1752: Limit amount of data read by limiting the call to readline(). The SSLFakeFile.readline() method needs to support limiting readline() as well. It's not a full emulation of readline()'s signature, but this class is only used by smtplib's code, so it doesn't have to be. Modified version of original patch by Christian Heimes. --- Lib/smtplib.py | 13 ++++++++++--- Lib/test/test_smtplib.py | 29 ++++++++++++++++++++++++++++- Misc/NEWS | 5 ++++- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 41a3c97b626..a8a7884482c 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -57,6 +57,7 @@ SMTP_PORT = 25 SMTP_SSL_PORT = 465 CRLF="\r\n" +_MAXLINE = 8192 # more than 8 times larger than RFC 821, 4.5.3 OLDSTYLE_AUTH = re.compile(r"auth=(.*)", re.I) @@ -170,10 +171,14 @@ class SSLFakeFile: def __init__(self, sslobj): self.sslobj = sslobj - def readline(self): + def readline(self, size=-1): + if size < 0: + size = None str = "" chr = None while chr != "\n": + if size is not None and len(str) >= size: + break chr = self.sslobj.read(1) if not chr: break str += chr @@ -334,11 +339,13 @@ def getreply(self): if self.file is None: self.file = self.sock.makefile('rb') while 1: - line = self.file.readline() + line = self.file.readline(_MAXLINE + 1) if line == '': self.close() raise SMTPServerDisconnected("Connection unexpectedly closed") - if self.debuglevel > 0: print>>stderr, 'reply:', repr(line) + if self.debuglevel > 0: print >>stderr, 'reply:', repr(line) + if len(line) > _MAXLINE: + raise SMTPResponseException(500, "Line too long.") resp.append(line[4:].strip()) code=line[:3] # Check that the error code is syntactically correct. diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 644cbf7be6f..5b2cf54a3a2 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -273,6 +273,32 @@ def testFailingHELO(self): HOST, self.port, 'localhost', 3) +class TooLongLineTests(TestCase): + respdata = '250 OK' + ('.' * smtplib._MAXLINE * 2) + '\n' + + def setUp(self): + self.old_stdout = sys.stdout + self.output = StringIO.StringIO() + sys.stdout = self.output + + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(15) + self.port = test_support.bind_port(self.sock) + servargs = (self.evt, self.respdata, self.sock) + threading.Thread(target=server, args=servargs).start() + self.evt.wait() + self.evt.clear() + + def tearDown(self): + self.evt.wait() + sys.stdout = self.old_stdout + + def testLineTooLong(self): + self.assertRaises(smtplib.SMTPResponseException, smtplib.SMTP, + HOST, self.port, 'localhost', 3) + + sim_users = {'Mr.A@somewhere.com':'John A', 'Ms.B@somewhere.com':'Sally B', 'Mrs.C@somewhereesle.com':'Ruth C', @@ -482,7 +508,8 @@ def testAUTH_CRAM_MD5(self): def test_main(verbose=None): test_support.run_unittest(GeneralTests, DebuggingServerTests, NonConnectingTests, - BadHELOServerTests, SMTPSimTests) + BadHELOServerTests, SMTPSimTests, + TooLongLineTests) if __name__ == '__main__': test_main() diff --git a/Misc/NEWS b/Misc/NEWS index 790dc583b36..de8c2021f26 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -16,13 +16,16 @@ Library - Issue #16248: Disable code execution from the user's home directory by tkinter when the -E flag is passed to Python. Patch by Zachary Ware. +- Issue #16042: CVE-2013-1752: smtplib: Limit amount of data read by + limiting the call to readline(). Original patch by Christian Heimes. + Extension Modules ----------------- - Issue #18709: Fix CVE-2013-4238. The SSL module now handles NULL bytes inside subjectAltName correctly. Formerly the module has used OpenSSL's GENERAL_NAME_print() function to get the string represention of ASN.1 - strings for `rfc822Name` (email), `dNSName` (DNS) and + strings for `rfc822Name` (email), `dNSName` (DNS) and `uniformResourceIdentifier` (URI). From 0b2a5edb9e76b671607188a788215a1594d1a0a9 Mon Sep 17 00:00:00 2001 From: Eli Bendersky Date: Sun, 15 Sep 2013 12:08:14 -0700 Subject: [PATCH 373/480] Close #18945: Add tests for tempfile name collision handling. Patch by Vlad Shcherbina --- Lib/test/test_tempfile.py | 87 +++++++++++++++++++++++++++++---------- 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index eb3e2b9a6dd..011e7d0e1cc 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -8,6 +8,7 @@ import sys import re import warnings +import contextlib import unittest from test import test_support as support @@ -270,6 +271,22 @@ def test_same_thing(self): test_classes.append(test__get_candidate_names) +@contextlib.contextmanager +def _inside_empty_temp_dir(): + dir = tempfile.mkdtemp() + try: + with support.swap_attr(tempfile, 'tempdir', dir): + yield + finally: + support.rmtree(dir) + + +def _mock_candidate_names(*names): + return support.swap_attr(tempfile, + '_get_candidate_names', + lambda: iter(names)) + + class test__mkstemp_inner(TC): """Test the internal function _mkstemp_inner.""" @@ -386,31 +403,36 @@ def test_textmode(self): self.do_create(bin=0).write("blat\n") # XXX should test that the file really is a text file + def default_mkstemp_inner(self): + return tempfile._mkstemp_inner(tempfile.gettempdir(), + tempfile.template, + '', + tempfile._bin_openflags) + + def test_collision_with_existing_file(self): + # _mkstemp_inner tries another name when a file with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + (fd1, name1) = self.default_mkstemp_inner() + os.close(fd1) + self.assertTrue(name1.endswith('aaa')) + + (fd2, name2) = self.default_mkstemp_inner() + os.close(fd2) + self.assertTrue(name2.endswith('bbb')) + def test_collision_with_existing_directory(self): # _mkstemp_inner tries another name when a directory with # the chosen name already exists - container_dir = tempfile.mkdtemp() - try: - def mock_get_candidate_names(): - return iter(['aaa', 'aaa', 'bbb']) - with support.swap_attr(tempfile, - '_get_candidate_names', - mock_get_candidate_names): - dir = tempfile.mkdtemp(dir=container_dir) - self.assertTrue(dir.endswith('aaa')) - - flags = tempfile._bin_openflags - (fd, name) = tempfile._mkstemp_inner(container_dir, - tempfile.template, - '', - flags) - try: - self.assertTrue(name.endswith('bbb')) - finally: - os.close(fd) - os.unlink(name) - finally: - support.rmtree(container_dir) + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + dir = tempfile.mkdtemp() + self.assertTrue(dir.endswith('aaa')) + + (fd, name) = self.default_mkstemp_inner() + os.close(fd) + self.assertTrue(name.endswith('bbb')) test_classes.append(test__mkstemp_inner) @@ -587,6 +609,27 @@ def test_mode(self): finally: os.rmdir(dir) + def test_collision_with_existing_file(self): + # mkdtemp tries another name when a file with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + file = tempfile.NamedTemporaryFile(delete=False) + file.close() + self.assertTrue(file.name.endswith('aaa')) + dir = tempfile.mkdtemp() + self.assertTrue(dir.endswith('bbb')) + + def test_collision_with_existing_directory(self): + # mkdtemp tries another name when a directory with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + dir1 = tempfile.mkdtemp() + self.assertTrue(dir1.endswith('aaa')) + dir2 = tempfile.mkdtemp() + self.assertTrue(dir2.endswith('bbb')) + test_classes.append(test_mkdtemp) From 8f3680ae1ed52c682e88bd12dff3325955784f27 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Mon, 16 Sep 2013 13:48:44 -0400 Subject: [PATCH 374/480] #14984: On POSIX, enforce permissions when reading default .netrc. Initial patch by Bruno Piguet. This is implemented as if a useful .netrc file could exist without passwords, which is possible in the general case; but in fact our netrc implementation does not support it. Fixing that issue will be an enhancement. --- Doc/library/netrc.rst | 6 ++++++ Lib/netrc.py | 23 ++++++++++++++++++++++- Lib/test/test_netrc.py | 23 ++++++++++++++++++++++- Misc/NEWS | 6 ++++++ 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 8a2f1c6088f..97927998fab 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -21,6 +21,12 @@ the Unix :program:`ftp` program and other FTP clients. no argument is given, the file :file:`.netrc` in the user's home directory will be read. Parse errors will raise :exc:`NetrcParseError` with diagnostic information including the file name, line number, and terminating token. + If no argument is specified on a POSIX system, the presence of passwords in + the :file:`.netrc` file will raise a :exc:`NetrcParseError` if the file + ownership or permissions are insecure (owned by a user other than the user + running the process, or accessible for read or write by any other user). + This implements security behavior equivalent to that of ftp and other + programs that use :file:`.netrc`. .. exception:: NetrcParseError diff --git a/Lib/netrc.py b/Lib/netrc.py index 5493d77d3e8..0b4eedff4ee 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -2,7 +2,7 @@ # Module and documentation by Eric S. Raymond, 21 Dec 1998 -import os, shlex +import os, stat, shlex, pwd __all__ = ["netrc", "NetrcParseError"] @@ -21,6 +21,7 @@ def __str__(self): class netrc: def __init__(self, file=None): + default_netrc = file is None if file is None: try: file = os.path.join(os.environ['HOME'], ".netrc") @@ -77,6 +78,26 @@ def __init__(self, file=None): elif tt == 'account': account = lexer.get_token() elif tt == 'password': + if os.name == 'posix' and default_netrc: + prop = os.fstat(fp.fileno()) + if prop.st_uid != os.getuid(): + try: + fowner = pwd.getpwuid(prop.st_uid)[0] + except KeyError: + fowner = 'uid %s' % prop.st_uid + try: + user = pwd.getpwuid(os.getuid())[0] + except KeyError: + user = 'uid %s ' % os.getuid() + raise NetrcParseError( + ("~/.netrc file owner (%s) does not match" + " current user (%s)") % (fowner, user), + file, lexer.lineno) + if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)): + raise NetrcParseError( + "~/.netrc access too permissive: access" + " permissions must restrict access to only" + " the owner", file, lexer.lineno) password = lexer.get_token() else: raise NetrcParseError("bad follower token %r" % tt, diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index b536255aba3..24ad78688c7 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -32,7 +32,7 @@ def setUp (self): def tearDown (self): del self.netrc - os.unlink(temp_filename) + test_support.unlink(temp_filename) def test_case_1(self): self.assert_(self.netrc.macros == {'macro1':['line1\n', 'line2\n'], @@ -41,6 +41,27 @@ def test_case_1(self): self.assert_(self.netrc.hosts['foo'] == ('log1', 'acct1', 'pass1')) self.assert_(self.netrc.hosts['default'] == ('log2', None, 'pass2')) + if os.name == 'posix': + def test_security(self): + # This test is incomplete since we are normally not run as root and + # therefore can't test the file ownership being wrong. + os.unlink(temp_filename) + d = test_support.TESTFN + try: + os.mkdir(d) + fn = os.path.join(d, '.netrc') + with open(fn, 'wt') as f: + f.write(TEST_NETRC) + with test_support.EnvironmentVarGuard() as environ: + environ.set('HOME', d) + os.chmod(fn, 0600) + self.netrc = netrc.netrc() + self.test_case_1() + os.chmod(fn, 0622) + self.assertRaises(netrc.NetrcParseError, netrc.netrc) + finally: + test_support.rmtree(d) + def test_main(): test_support.run_unittest(NetrcTestCase) diff --git a/Misc/NEWS b/Misc/NEWS index de8c2021f26..833cd05fa82 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,12 @@ Core and Builtins Library ------- +- Issue #14984: On POSIX systems, when netrc is called without a filename + argument (and therefore is reading the user's $HOME/.netrc file), it now + enforces the same security rules as typical ftp clients: the .netrc file must + be owned by the user that owns the process and must not be readable by any + other user. + - Issue #16248: Disable code execution from the user's home directory by tkinter when the -E flag is passed to Python. Patch by Zachary Ware. From 69ce16e264b82ac76fba092590adf73d885f612d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 17 Sep 2013 00:00:46 +0300 Subject: [PATCH 375/480] Issue #18873: IDLE, 2to3, and the findnocoding.py script now detect Python source code encoding only in comment lines. --- Lib/idlelib/IOBinding.py | 13 +++++++------ Lib/lib2to3/pgen2/tokenize.py | 9 ++++----- Lib/lib2to3/tests/data/false_encoding.py | 2 ++ Lib/lib2to3/tests/test_refactor.py | 4 ++++ Misc/NEWS | 6 ++++++ Tools/scripts/findnocoding.py | 6 +++--- 6 files changed, 26 insertions(+), 14 deletions(-) create mode 100644 Lib/lib2to3/tests/data/false_encoding.py diff --git a/Lib/idlelib/IOBinding.py b/Lib/idlelib/IOBinding.py index ba45ee833fe..f5665b9dd00 100644 --- a/Lib/idlelib/IOBinding.py +++ b/Lib/idlelib/IOBinding.py @@ -71,7 +71,7 @@ encoding = encoding.lower() -coding_re = re.compile("coding[:=]\s*([-\w_.]+)") +coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) class EncodingMessage(SimpleDialog): "Inform user that an encoding declaration is needed." @@ -125,11 +125,12 @@ def coding_spec(str): Raise LookupError if the encoding is declared but unknown. """ # Only consider the first two lines - str = str.split("\n")[:2] - str = "\n".join(str) - - match = coding_re.search(str) - if not match: + str = str.split("\n", 2)[:2] + for line in lst: + match = coding_re.match(line) + if match is not None: + break + else: return None name = match.group(1) # Check whether the encoding is known diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index e090aa98ddb..9cf2a69dc55 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -236,7 +236,7 @@ def compat(self, token, iterable): startline = False toks_append(tokval) -cookie_re = re.compile("coding[:=]\s*([-\w.]+)") +cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) def _get_normal_name(orig_enc): """Imitates get_normal_name in tokenizer.c.""" @@ -281,11 +281,10 @@ def find_cookie(line): line_string = line.decode('ascii') except UnicodeDecodeError: return None - - matches = cookie_re.findall(line_string) - if not matches: + match = cookie_re.match(line_string) + if not match: return None - encoding = _get_normal_name(matches[0]) + encoding = _get_normal_name(match.group(1)) try: codec = lookup(encoding) except LookupError: diff --git a/Lib/lib2to3/tests/data/false_encoding.py b/Lib/lib2to3/tests/data/false_encoding.py new file mode 100644 index 00000000000..f4e59e787da --- /dev/null +++ b/Lib/lib2to3/tests/data/false_encoding.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +print '#coding=0' diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py index 6020d1ff18b..543fd421366 100644 --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -271,6 +271,10 @@ def test_file_encoding(self): fn = os.path.join(TEST_DATA_DIR, "different_encoding.py") self.check_file_refactoring(fn) + def test_false_file_encoding(self): + fn = os.path.join(TEST_DATA_DIR, "false_encoding.py") + data = self.check_file_refactoring(fn) + def test_bom(self): fn = os.path.join(TEST_DATA_DIR, "bom.py") data = self.check_file_refactoring(fn) diff --git a/Misc/NEWS b/Misc/NEWS index d9b413d9964..7f4ef6024d9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -183,6 +183,9 @@ Library Tools/Demos ----------- +- Issue #18873: 2to3 and the findnocoding.py script now detect Python source + code encoding only in comment lines. + - Issue #18817: Fix a resource warning in Lib/aifc.py demo. - Issue #18439: Make patchcheck work on Windows for ACKS, NEWS. @@ -207,6 +210,9 @@ Build IDLE ---- +- Issue #18873: IDLE now detects Python source code encoding only in comment + lines. + - Issue #18988: The "Tab" key now works when a word is already autocompleted. - Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster. diff --git a/Tools/scripts/findnocoding.py b/Tools/scripts/findnocoding.py index e49fc425ffc..838e5733538 100755 --- a/Tools/scripts/findnocoding.py +++ b/Tools/scripts/findnocoding.py @@ -32,13 +32,13 @@ def walk_python_files(self, paths, *args, **kwargs): "no sophisticated Python source file search will be done.") -decl_re = re.compile(r"coding[=:]\s*([-\w.]+)") +decl_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) def get_declaration(line): - match = decl_re.search(line) + match = decl_re.match(line) if match: return match.group(1) - return '' + return b'' def has_correct_encoding(text, codec): try: From e00c22c1f20da0ed43e2b5252cccb96b25f6b363 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 17 Sep 2013 10:09:08 +0300 Subject: [PATCH 376/480] Remove the use of non-existing re.ASCII. (fixes a regression in 3d46ef0c62c5, issue #18873) --- Lib/idlelib/IOBinding.py | 2 +- Lib/lib2to3/pgen2/tokenize.py | 2 +- Tools/scripts/findnocoding.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/idlelib/IOBinding.py b/Lib/idlelib/IOBinding.py index f5665b9dd00..2eae11b5411 100644 --- a/Lib/idlelib/IOBinding.py +++ b/Lib/idlelib/IOBinding.py @@ -71,7 +71,7 @@ encoding = encoding.lower() -coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) +coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') class EncodingMessage(SimpleDialog): "Inform user that an encoding declaration is needed." diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 9cf2a69dc55..f6e0284c2f5 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -236,7 +236,7 @@ def compat(self, token, iterable): startline = False toks_append(tokval) -cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) +cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') def _get_normal_name(orig_enc): """Imitates get_normal_name in tokenizer.c.""" diff --git a/Tools/scripts/findnocoding.py b/Tools/scripts/findnocoding.py index 838e5733538..5d932908d36 100755 --- a/Tools/scripts/findnocoding.py +++ b/Tools/scripts/findnocoding.py @@ -32,7 +32,7 @@ def walk_python_files(self, paths, *args, **kwargs): "no sophisticated Python source file search will be done.") -decl_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) +decl_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') def get_declaration(line): match = decl_re.match(line) From d1861e52a725523809eb27028f374769b26cd842 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Tue, 17 Sep 2013 20:08:09 -0400 Subject: [PATCH 377/480] Add versionchanged for #14984, remove extra blank from string. --- Doc/library/netrc.rst | 2 ++ Lib/netrc.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 97927998fab..1b53efc542a 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -28,6 +28,8 @@ the Unix :program:`ftp` program and other FTP clients. This implements security behavior equivalent to that of ftp and other programs that use :file:`.netrc`. + .. versionchanged:: 2.6.9 Added the POSIX permissions check. + .. exception:: NetrcParseError diff --git a/Lib/netrc.py b/Lib/netrc.py index 0b4eedff4ee..2b45bc2518a 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -88,7 +88,7 @@ def __init__(self, file=None): try: user = pwd.getpwuid(os.getuid())[0] except KeyError: - user = 'uid %s ' % os.getuid() + user = 'uid %s' % os.getuid() raise NetrcParseError( ("~/.netrc file owner (%s) does not match" " current user (%s)") % (fowner, user), From c1361683c2c19d57088c47358b934d19304c6597 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Wed, 18 Sep 2013 08:35:45 -0400 Subject: [PATCH 378/480] #19037: adjust file times *before* moving maildir files into place. This avoids race conditions when other programs are monitoring the maildir directory. Patch by janzert. --- Lib/mailbox.py | 18 +++++++++++++----- Misc/NEWS | 4 ++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 530d3c5a66e..4cdd7d45ea1 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -286,6 +286,12 @@ def add(self, message): suffix = '' uniq = os.path.basename(tmp_file.name).split(self.colon)[0] dest = os.path.join(self._path, subdir, uniq + suffix) + if isinstance(message, MaildirMessage): + os.utime(tmp_file.name, + (os.path.getatime(tmp_file.name), message.get_date())) + # No file modification should be done after the file is moved to its + # final position in order to prevent race conditions with changes + # from other programs try: if hasattr(os, 'link'): os.link(tmp_file.name, dest) @@ -299,8 +305,6 @@ def add(self, message): % dest) else: raise - if isinstance(message, MaildirMessage): - os.utime(dest, (os.path.getatime(dest), message.get_date())) return uniq def remove(self, key): @@ -335,11 +339,15 @@ def __setitem__(self, key, message): else: suffix = '' self.discard(key) + tmp_path = os.path.join(self._path, temp_subpath) new_path = os.path.join(self._path, subdir, key + suffix) - os.rename(os.path.join(self._path, temp_subpath), new_path) if isinstance(message, MaildirMessage): - os.utime(new_path, (os.path.getatime(new_path), - message.get_date())) + os.utime(tmp_path, + (os.path.getatime(tmp_path), message.get_date())) + # No file modification should be done after the file is moved to its + # final position in order to prevent race conditions with changes + # from other programs + os.rename(tmp_path, new_path) def get_message(self, key): """Return a Message representation or raise a KeyError.""" diff --git a/Misc/NEWS b/Misc/NEWS index 7f4ef6024d9..3c44f00ec0f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,10 @@ Core and Builtins Library ------- +- Issue #19037: The mailbox module now makes all changes to maildir files + before moving them into place, to avoid race conditions with other programs + that may be accessing the maildir directory. + - Issue #14984: On POSIX systems, when netrc is called without a filename argument (and therefore is reading the user's $HOME/.netrc file), it now enforces the same security rules as typical ftp clients: the .netrc file must From 4a19682de789fb03579f462a6c964ca2f58d6e79 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Wed, 18 Sep 2013 08:49:25 -0400 Subject: [PATCH 379/480] #14984: only import pwd on POSIX. --- Lib/netrc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/netrc.py b/Lib/netrc.py index 2b45bc2518a..ce6f4ddb976 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -2,7 +2,9 @@ # Module and documentation by Eric S. Raymond, 21 Dec 1998 -import os, stat, shlex, pwd +import os, stat, shlex +if os.name == 'posix': + import pwd __all__ = ["netrc", "NetrcParseError"] From 0209e170cceabd2cffc8d4b219805e8b20d32693 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 20 Sep 2013 21:25:53 +0300 Subject: [PATCH 380/480] Issue #18050: Fixed an incompatibility of the re module with Python 2.7.3 and older binaries. --- Lib/sre_compile.py | 1 - Lib/sre_constants.py | 6 +++++- Lib/sre_parse.py | 1 - Misc/NEWS | 3 +++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py index 15d23243c16..97c1663b0b9 100644 --- a/Lib/sre_compile.py +++ b/Lib/sre_compile.py @@ -13,7 +13,6 @@ import _sre, sys import sre_parse from sre_constants import * -from _sre import MAXREPEAT assert _sre.MAGIC == MAGIC, "SRE module mismatch" diff --git a/Lib/sre_constants.py b/Lib/sre_constants.py index b0175e71be6..69224e274c5 100644 --- a/Lib/sre_constants.py +++ b/Lib/sre_constants.py @@ -15,7 +15,11 @@ MAGIC = 20031017 -from _sre import MAXREPEAT +try: + from _sre import MAXREPEAT +except ImportError: + import _sre + MAXREPEAT = _sre.MAXREPEAT = 65535 # SRE standard exception (access as sre.error) # should this really be here? diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index e8d35a6d412..e37e2cfce3f 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -15,7 +15,6 @@ import sys from sre_constants import * -from _sre import MAXREPEAT SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" diff --git a/Misc/NEWS b/Misc/NEWS index 3c44f00ec0f..61832f831cf 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #18050: Fixed an incompatibility of the re module with Python 2.7.3 + and older binaries. + - Issue #19037: The mailbox module now makes all changes to maildir files before moving them into place, to avoid race conditions with other programs that may be accessing the maildir directory. From 9749f3a075faaa04140260bde1b6707a92661188 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 22 Sep 2013 19:36:09 +1000 Subject: [PATCH 381/480] This check can still fail on RHEL6 --- Lib/test/test_gdb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 5302b5adb42..d1e656704bd 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -53,13 +53,13 @@ def run_gdb(*args, **env_vars): if not gdbpy_version: raise unittest.SkipTest("gdb not built with embedded python support") -# Verify that "gdb" can load our custom hooks. In theory this should never -# fail, but we don't handle the case of the hooks file not existing if the -# tests are run from an installed Python (we'll produce failures in that case). +# Verify that "gdb" can load our custom hooks, as OS security settings may +# disallow this without a customised .gdbinit. cmd = ['--args', sys.executable] _, gdbpy_errors = run_gdb('--args', sys.executable) if "auto-loading has been declined" in gdbpy_errors: msg = "gdb security settings prevent use of custom hooks: " + raise unittest.SkipTest(msg + gdbpy_errors.rstrip()) def python_is_optimized(): cflags = sysconfig.get_config_vars()['PY_CFLAGS'] From d89c828abfbcd7f60247644adb8d088c7a509875 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 22 Sep 2013 11:43:10 +0200 Subject: [PATCH 382/480] Closes #19061: make shelve security warning consistent between 2.x and 3.x. --- Doc/library/shelve.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index ad36b458969..b02f763dc36 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -47,9 +47,11 @@ lots of shared sub-objects. The keys are ordinary strings. Like file objects, shelve objects should be closed explicitly to ensure that the persistent data is flushed to disk. - Since the :mod:`shelve` module stores objects using :mod:`pickle`, the same - security precautions apply. Accordingly, you should avoid loading a shelf - from an untrusted source. +.. warning:: + + Because the :mod:`shelve` module is backed by :mod:`pickle`, it is insecure + to load a shelf from an untrusted source. Like with pickle, loading a shelf + can execute arbitrary code. Shelf objects support all methods supported by dictionaries. This eases the transition from dictionary based scripts to those requiring persistent storage. From 67952b6d8403db1ac6ba32e7f2c0d4c6cebee37d Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 22 Sep 2013 11:45:02 +0200 Subject: [PATCH 383/480] Closes #19043: remove detailed listing of versions from license files Since all versions since 2.2 are under the same licensing terms, this saves the release manager from touching the two files for every new minor release. --- Doc/license.rst | 50 +------------------------------------------------ LICENSE | 27 +------------------------- 2 files changed, 2 insertions(+), 75 deletions(-) diff --git a/Doc/license.rst b/Doc/license.rst index 7b519a9b990..2f8546eb8c4 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -50,59 +50,11 @@ been GPL-compatible; the table below summarizes the various releases. +----------------+--------------+-----------+------------+-----------------+ | 2.1.1 | 2.1+2.0.1 | 2001 | PSF | yes | +----------------+--------------+-----------+------------+-----------------+ -| 2.2 | 2.1.1 | 2001 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ | 2.1.2 | 2.1.1 | 2002 | PSF | yes | +----------------+--------------+-----------+------------+-----------------+ | 2.1.3 | 2.1.2 | 2002 | PSF | yes | +----------------+--------------+-----------+------------+-----------------+ -| 2.2.1 | 2.2 | 2002 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.2.2 | 2.2.1 | 2002 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.2.3 | 2.2.2 | 2002-2003 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.3 | 2.2.2 | 2002-2003 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.3.1 | 2.3 | 2002-2003 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.3.2 | 2.3.1 | 2003 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.3.3 | 2.3.2 | 2003 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.3.4 | 2.3.3 | 2004 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.3.5 | 2.3.4 | 2005 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.4 | 2.3 | 2004 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.4.1 | 2.4 | 2005 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.4.2 | 2.4.1 | 2005 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.4.3 | 2.4.2 | 2006 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.4.4 | 2.4.3 | 2006 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.5 | 2.4 | 2006 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.5.1 | 2.5 | 2007 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.5.2 | 2.5.1 | 2008 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.5.3 | 2.5.2 | 2008 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.6 | 2.5 | 2008 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.6.1 | 2.6 | 2008 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.6.2 | 2.6.1 | 2009 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.6.3 | 2.6.2 | 2009 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.6.4 | 2.6.3 | 2010 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.7 | 2.6 | 2010 | PSF | yes | +| 2.2 and above | 2.1.1 | 2001-now | PSF | yes | +----------------+--------------+-----------+------------+-----------------+ .. note:: diff --git a/LICENSE b/LICENSE index 2e89c09a244..56a5d5cc7f1 100644 --- a/LICENSE +++ b/LICENSE @@ -36,34 +36,9 @@ the various releases. 2.1 2.0+1.6.1 2001 PSF no 2.0.1 2.0+1.6.1 2001 PSF yes 2.1.1 2.1+2.0.1 2001 PSF yes - 2.2 2.1.1 2001 PSF yes 2.1.2 2.1.1 2002 PSF yes 2.1.3 2.1.2 2002 PSF yes - 2.2.1 2.2 2002 PSF yes - 2.2.2 2.2.1 2002 PSF yes - 2.2.3 2.2.2 2003 PSF yes - 2.3 2.2.2 2002-2003 PSF yes - 2.3.1 2.3 2002-2003 PSF yes - 2.3.2 2.3.1 2002-2003 PSF yes - 2.3.3 2.3.2 2002-2003 PSF yes - 2.3.4 2.3.3 2004 PSF yes - 2.3.5 2.3.4 2005 PSF yes - 2.4 2.3 2004 PSF yes - 2.4.1 2.4 2005 PSF yes - 2.4.2 2.4.1 2005 PSF yes - 2.4.3 2.4.2 2006 PSF yes - 2.4.4 2.4.3 2006 PSF yes - 2.5 2.4 2006 PSF yes - 2.5.1 2.5 2007 PSF yes - 2.5.2 2.5.1 2008 PSF yes - 2.5.3 2.5.2 2008 PSF yes - 2.6 2.5 2008 PSF yes - 2.6.1 2.6 2008 PSF yes - 2.6.2 2.6.1 2009 PSF yes - 2.6.3 2.6.2 2009 PSF yes - 2.6.4 2.6.3 2009 PSF yes - 2.6.5 2.6.4 2010 PSF yes - 2.7 2.6 2010 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes Footnotes: From 1874f4708d9203e70be3f486171c93c859581bdd Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sun, 22 Sep 2013 16:07:09 -0400 Subject: [PATCH 384/480] - Issue #16039: CVE-2013-1752: Change use of readline in imaplib module to limit line length. Patch by Emil Lind. --- Lib/imaplib.py | 14 +++++++++++++- Lib/test/test_imaplib.py | 10 ++++++++++ Misc/NEWS | 3 +++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 8734a843041..bac4c05cb30 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -35,6 +35,15 @@ IMAP4_SSL_PORT = 993 AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first +# Maximal line length when calling readline(). This is to prevent +# reading arbitrary length lines. RFC 3501 and 2060 (IMAP 4rev1) +# don't specify a line length. RFC 2683 however suggests limiting client +# command lines to 1000 octets and server command lines to 8000 octets. +# We have selected 10000 for some extra margin and since that is supposedly +# also what UW and Panda IMAP does. +_MAXLINE = 10000 + + # Commands Commands = { @@ -237,7 +246,10 @@ def read(self, size): def readline(self): """Read line from remote.""" - return self.file.readline() + line = self.file.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise self.error("got more than %d bytes" % _MAXLINE) + return line def send(self, data): diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index f288ef0f979..ba36235b19b 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -176,6 +176,16 @@ def handle(self): self.assertRaises(imaplib.IMAP4.abort, self.imap_class, *server.server_address) + def test_linetoolong(self): + class TooLongHandler(TimeoutStreamRequestHandler): + def handle(self): + # Send a very long response line + self.wfile.write('* OK ' + imaplib._MAXLINE*'x' + '\r\n') + + with self.reaped_server(TooLongHandler) as server: + self.assertRaises(imaplib.IMAP4.error, + self.imap_class, *server.server_address) + class ThreadedNetworkedTests(BaseThreadedNetworkedTests): server_class = SocketServer.TCPServer diff --git a/Misc/NEWS b/Misc/NEWS index 833cd05fa82..e6c60b9febc 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins Library ------- +- Issue #16039: CVE-2013-1752: Change use of readline in imaplib module to + limit line length. Patch by Emil Lind. + - Issue #14984: On POSIX systems, when netrc is called without a filename argument (and therefore is reading the user's $HOME/.netrc file), it now enforces the same security rules as typical ftp clients: the .netrc file must From 8cc020dcfb3d71840b12357c111ebae789b46a36 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 25 Sep 2013 09:36:58 -0400 Subject: [PATCH 385/480] =?UTF-8?q?-=20Issue=20#16038:=20CVE-2013-1752:=20?= =?UTF-8?q?ftplib:=20Limit=20amount=20of=20data=20read=20by=20=20=20limiti?= =?UTF-8?q?ng=20the=20call=20to=20readline().=20=20Original=20patch=20by?= =?UTF-8?q?=20Micha=C5=82=20=20=20Jastrz=C4=99bski=20and=20Giampaolo=20Rod?= =?UTF-8?q?ola.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit with test fixes by Serhiy Storchaka. --- Lib/ftplib.py | 15 ++++++++++++--- Lib/test/test_ftplib.py | 22 +++++++++++++++++++++- Misc/NEWS | 4 ++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 22decb2dbb5..a3c647538de 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -54,6 +54,8 @@ # The standard FTP server control port FTP_PORT = 21 +# The sizehint parameter passed to readline() calls +MAXLINE = 8192 # Exception raised when an error or invalid response is received @@ -100,6 +102,7 @@ class FTP: debugging = 0 host = '' port = FTP_PORT + maxline = MAXLINE sock = None file = None welcome = None @@ -179,7 +182,9 @@ def putcmd(self, line): # Internal: return one line from the server, stripping CRLF. # Raise EOFError if the connection is closed def getline(self): - line = self.file.readline() + line = self.file.readline(self.maxline + 1) + if len(line) > self.maxline: + raise Error("got more than %d bytes" % self.maxline) if self.debugging > 1: print '*get*', self.sanitize(line) if not line: raise EOFError @@ -421,7 +426,9 @@ def retrlines(self, cmd, callback = None): conn = self.transfercmd(cmd) fp = conn.makefile('rb') while 1: - line = fp.readline() + line = fp.readline(self.maxline + 1) + if len(line) > self.maxline: + raise Error("got more than %d bytes" % self.maxline) if self.debugging > 2: print '*retr*', repr(line) if not line: break @@ -473,7 +480,9 @@ def storlines(self, cmd, fp, callback=None): self.voidcmd('TYPE A') conn = self.transfercmd(cmd) while 1: - buf = fp.readline() + buf = fp.readline(self.maxline + 1) + if len(buf) > self.maxline: + raise Error("got more than %d bytes" % self.maxline) if not buf: break if buf[-2:] != CRLF: if buf[-1] in CRLF: buf = buf[:-1] diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 1c2ceeb8728..2d5a814db2d 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -46,6 +46,7 @@ def __init__(self, conn): self.last_received_cmd = None self.last_received_data = '' self.next_response = '' + self.next_retr_data = RETR_DATA self.push('220 welcome') def collect_incoming_data(self, data): @@ -162,7 +163,7 @@ def cmd_stor(self, arg): def cmd_retr(self, arg): self.push('125 retr ok') - self.dtp.push(RETR_DATA) + self.dtp.push(self.next_retr_data) self.dtp.close_when_done() def cmd_list(self, arg): @@ -175,6 +176,11 @@ def cmd_nlst(self, arg): self.dtp.push(NLST_DATA) self.dtp.close_when_done() + def cmd_setlongretr(self, arg): + # For testing. Next RETR will return long line. + self.next_retr_data = 'x' * int(arg) + self.push('125 setlongretr ok') + class DummyFTPServer(asyncore.dispatcher, threading.Thread): @@ -362,6 +368,20 @@ def test_makepasv(self): # IPv4 is in use, just make sure send_epsv has not been used self.assertEqual(self.server.handler.last_received_cmd, 'pasv') + def test_line_too_long(self): + self.assertRaises(ftplib.Error, self.client.sendcmd, + 'x' * self.client.maxline * 2) + + def test_retrlines_too_long(self): + self.client.sendcmd('SETLONGRETR %d' % (self.client.maxline * 2)) + received = [] + self.assertRaises(ftplib.Error, + self.client.retrlines, 'retr', received.append) + + def test_storlines_too_long(self): + f = StringIO.StringIO('x' * self.client.maxline * 2) + self.assertRaises(ftplib.Error, self.client.storlines, 'stor', f) + class TestIPv6Environment(TestCase): diff --git a/Misc/NEWS b/Misc/NEWS index e6c60b9febc..8a0db6e1ca7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,10 @@ Core and Builtins Library ------- +- Issue #16038: CVE-2013-1752: ftplib: Limit amount of data read by + limiting the call to readline(). Original patch by MichaÅ‚ + JastrzÄ™bski and Giampaolo Rodola. + - Issue #16039: CVE-2013-1752: Change use of readline in imaplib module to limit line length. Patch by Emil Lind. From 88fb4ecbe93411a9351077163e960f64b394699a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 28 Sep 2013 21:31:36 +0300 Subject: [PATCH 386/480] Issue #18950: Fix miscellaneous bugs in the sunau module. Au_read.readframes() now updates current file position and reads correct number of frames from multichannel stream. Au_write.writeframesraw() now correctly updates current file position. Au_read and Au_write now correctly work with file object if start file position is not a zero. --- Lib/sunau.py | 25 ++++++++++++++++++++----- Misc/NEWS | 6 ++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Lib/sunau.py b/Lib/sunau.py index a04d8c01314..4f0885aaf56 100644 --- a/Lib/sunau.py +++ b/Lib/sunau.py @@ -203,6 +203,10 @@ def initfp(self, file): break else: self._info = '' + try: + self._data_pos = file.tell() + except (AttributeError, IOError): + self._data_pos = None def getfp(self): return self._file @@ -255,7 +259,8 @@ def readframes(self, nframes): if nframes == AUDIO_UNKNOWN_SIZE: data = self._file.read() else: - data = self._file.read(nframes * self._framesize * self._nchannels) + data = self._file.read(nframes * self._framesize) + self._soundpos += len(data) // self._framesize if self._encoding == AUDIO_FILE_ENCODING_MULAW_8: import audioop data = audioop.ulaw2lin(data, self._sampwidth) @@ -263,8 +268,10 @@ def readframes(self, nframes): return None # XXX--not implemented yet def rewind(self): + if self._data_pos is None: + raise IOError('cannot seek') + self._file.seek(self._data_pos) self._soundpos = 0 - self._file.seek(self._hdr_size) def tell(self): return self._soundpos @@ -272,7 +279,9 @@ def tell(self): def setpos(self, pos): if pos < 0 or pos > self.getnframes(): raise Error, 'position not in range' - self._file.seek(pos * self._framesize + self._hdr_size) + if self._data_pos is None: + raise IOError('cannot seek') + self._file.seek(self._data_pos + pos * self._framesize) self._soundpos = pos def close(self): @@ -382,10 +391,10 @@ def tell(self): def writeframesraw(self, data): self._ensure_header_written() - nframes = len(data) / self._framesize if self._comptype == 'ULAW': import audioop data = audioop.lin2ulaw(data, self._sampwidth) + nframes = len(data) // self._framesize self._file.write(data) self._nframeswritten = self._nframeswritten + nframes self._datawritten = self._datawritten + len(data) @@ -445,6 +454,10 @@ def _write_header(self): length = AUDIO_UNKNOWN_SIZE else: length = self._nframes * self._framesize + try: + self._form_length_pos = self._file.tell() + except (AttributeError, IOError): + self._form_length_pos = None _write_u32(self._file, length) self._datalength = length _write_u32(self._file, encoding) @@ -454,7 +467,9 @@ def _write_header(self): self._file.write('\0'*(header_size - len(self._info) - 24)) def _patchheader(self): - self._file.seek(8) + if self._form_length_pos is None: + raise IOError('cannot seek') + self._file.seek(self._form_length_pos) _write_u32(self._file, self._datawritten) self._datalength = self._datawritten self._file.seek(0, 2) diff --git a/Misc/NEWS b/Misc/NEWS index 61832f831cf..d9e23339435 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,12 @@ Core and Builtins Library ------- +- Issue #18950: Fix miscellaneous bugs in the sunau module. + Au_read.readframes() now updates current file position and reads correct + number of frames from multichannel stream. Au_write.writeframesraw() now + correctly updates current file position. Au_read and Au_write now correctly + work with file object if start file position is not a zero. + - Issue #18050: Fixed an incompatibility of the re module with Python 2.7.3 and older binaries. From 84183071b0d20878d80d8e3d6a1b81ffb6943a60 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 28 Sep 2013 15:12:37 -0400 Subject: [PATCH 387/480] fix duplicate test names (closes #19115) Patch by Xavier de Gaye. --- Lib/lib2to3/tests/test_fixers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index 15482812201..1817208c7c8 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -1405,27 +1405,27 @@ def test_27(self): a = "d.values()" self.check(b, a) - def test_14(self): + def test_28(self): b = "[i for i in d.viewkeys()]" a = "[i for i in d.keys()]" self.check(b, a) - def test_15(self): + def test_29(self): b = "(i for i in d.viewkeys())" a = "(i for i in d.keys())" self.check(b, a) - def test_17(self): + def test_30(self): b = "iter(d.viewkeys())" a = "iter(d.keys())" self.check(b, a) - def test_18(self): + def test_31(self): b = "list(d.viewkeys())" a = "list(d.keys())" self.check(b, a) - def test_19(self): + def test_32(self): b = "sorted(d.viewkeys())" a = "sorted(d.keys())" self.check(b, a) From df2ff178c2e7d51d1b9b2b37c33b67fceabfaf9f Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 29 Sep 2013 01:48:40 +0200 Subject: [PATCH 388/480] Issue #4366: Fix building extensions on all platforms when --enable-shared is used. --- Lib/distutils/command/build_ext.py | 8 +++----- Misc/NEWS | 3 +++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py index 923197bac43..f0a7d4cafdd 100644 --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -231,12 +231,10 @@ def finalize_options(self): # building python standard extensions self.library_dirs.append('.') - # for extensions under Linux or Solaris with a shared Python library, + # For building extensions with a shared Python library, # Python's library directory must be appended to library_dirs - sysconfig.get_config_var('Py_ENABLE_SHARED') - if ((sys.platform.startswith('linux') or sys.platform.startswith('gnu') - or sys.platform.startswith('sunos')) - and sysconfig.get_config_var('Py_ENABLE_SHARED')): + # See Issues: #1600860, #4366 + if (sysconfig.get_config_var('Py_ENABLE_SHARED')): if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) diff --git a/Misc/NEWS b/Misc/NEWS index d9e23339435..2984b57e43d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #4366: Fix building extensions on all platforms when --enable-shared + is used. + - Issue #18950: Fix miscellaneous bugs in the sunau module. Au_read.readframes() now updates current file position and reads correct number of frames from multichannel stream. Au_write.writeframesraw() now From 3430b6576b98b4c1ca3a296038141fec38714384 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 29 Sep 2013 11:08:04 -0400 Subject: [PATCH 389/480] move helper function into its test method (closes #19112) --- Lib/test/test_multiprocessing.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index 925f39dc4fa..dc4252ef872 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -2293,13 +2293,6 @@ def _ThisSubProcess(q): except Queue.Empty: pass -def _TestProcess(q): - queue = multiprocessing.Queue() - subProc = multiprocessing.Process(target=_ThisSubProcess, args=(queue,)) - subProc.daemon = True - subProc.start() - subProc.join() - def _afunc(x): return x*x @@ -2331,6 +2324,12 @@ def flush(self): class TestStdinBadfiledescriptor(unittest.TestCase): def test_queue_in_process(self): + def _TestProcess(q): + queue = multiprocessing.Queue() + subProc = multiprocessing.Process(target=_ThisSubProcess, args=(queue,)) + subProc.daemon = True + subProc.start() + subProc.join() queue = multiprocessing.Queue() proc = multiprocessing.Process(target=_TestProcess, args=(queue,)) proc.start() From bc00a3efde5d09a7252e4d2d415bf4b121c949c0 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 29 Sep 2013 11:13:27 -0400 Subject: [PATCH 390/480] condense two tests with the same name (closes #19114) --- Lib/distutils/tests/test_cmd.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/Lib/distutils/tests/test_cmd.py b/Lib/distutils/tests/test_cmd.py index e074099609a..51420c96cc0 100644 --- a/Lib/distutils/tests/test_cmd.py +++ b/Lib/distutils/tests/test_cmd.py @@ -34,6 +34,18 @@ def test_ensure_string_list(self): self.assertRaises(DistutilsOptionError, cmd.ensure_string_list, 'not_string_list2') + cmd.option1 = 'ok,dok' + cmd.ensure_string_list('option1') + self.assertEqual(cmd.option1, ['ok', 'dok']) + + cmd.option2 = ['xxx', 'www'] + cmd.ensure_string_list('option2') + + cmd.option3 = ['ok', 2] + self.assertRaises(DistutilsOptionError, cmd.ensure_string_list, + 'option3') + + def test_make_file(self): cmd = self.cmd @@ -77,19 +89,6 @@ def test_ensure_string(self): cmd.option3 = 1 self.assertRaises(DistutilsOptionError, cmd.ensure_string, 'option3') - def test_ensure_string_list(self): - cmd = self.cmd - cmd.option1 = 'ok,dok' - cmd.ensure_string_list('option1') - self.assertEqual(cmd.option1, ['ok', 'dok']) - - cmd.option2 = ['xxx', 'www'] - cmd.ensure_string_list('option2') - - cmd.option3 = ['ok', 2] - self.assertRaises(DistutilsOptionError, cmd.ensure_string_list, - 'option3') - def test_ensure_filename(self): cmd = self.cmd cmd.option1 = __file__ From 1bda4ea6cbdeca91ec431b9d8332869df2fbefc7 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 29 Sep 2013 11:15:31 -0400 Subject: [PATCH 391/480] remove duplicate method (closes #19127) --- Lib/xml/dom/minidom.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index f8ed232fb2e..c30e2462ee9 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -358,9 +358,6 @@ def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None, def _get_localName(self): return self.nodeName.split(":", 1)[-1] - def _get_name(self): - return self.name - def _get_specified(self): return self.specified From b118ae3553cd74683d15b2372dbdd5959ad413fb Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Sun, 29 Sep 2013 17:10:40 +0100 Subject: [PATCH 392/480] Issue #19112: avoid using function defined in method. --- Lib/test/test_multiprocessing.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index dc4252ef872..579229abca5 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -2287,12 +2287,19 @@ def test_pool_initializer(self): # Verifies os.close(sys.stdin.fileno) vs. sys.stdin.close() behavior # -def _ThisSubProcess(q): +def _this_sub_process(q): try: item = q.get(block=False) except Queue.Empty: pass +def _test_process(q): + queue = multiprocessing.Queue() + subProc = multiprocessing.Process(target=_this_sub_process, args=(queue,)) + subProc.daemon = True + subProc.start() + subProc.join() + def _afunc(x): return x*x @@ -2324,14 +2331,8 @@ def flush(self): class TestStdinBadfiledescriptor(unittest.TestCase): def test_queue_in_process(self): - def _TestProcess(q): - queue = multiprocessing.Queue() - subProc = multiprocessing.Process(target=_ThisSubProcess, args=(queue,)) - subProc.daemon = True - subProc.start() - subProc.join() queue = multiprocessing.Queue() - proc = multiprocessing.Process(target=_TestProcess, args=(queue,)) + proc = multiprocessing.Process(target=_test_process, args=(queue,)) proc.start() proc.join() From bddb07309958a93ccb1cd124878bc9fe370a54d9 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 29 Sep 2013 19:52:45 +0200 Subject: [PATCH 393/480] Properly initialize all fields of a SSL object after allocation. --- Misc/NEWS | 2 ++ Modules/_ssl.c | 1 + 2 files changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 2984b57e43d..af6ed7e96ca 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,8 @@ Core and Builtins Library ------- +- Properly initialize all fields of a SSL object after allocation. + - Issue #4366: Fix building extensions on all platforms when --enable-shared is used. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 9164a65201f..d95568cf167 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -286,6 +286,7 @@ newPySSLObject(PySocketSockObject *Sock, char *key_file, char *cert_file, self->ssl = NULL; self->ctx = NULL; self->Socket = NULL; + self->shutdown_seen_zero = 0; /* Make sure the SSL error state is initialized */ (void) ERR_get_state(); From fbd22119a8e433c66434c954e96493f7daf1c9f1 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sun, 29 Sep 2013 13:59:06 -0400 Subject: [PATCH 394/480] - Issue #16037: HTTPMessage.readheaders() raises an HTTPException when more than 100 headers are read. Adapted from patch by Jyrki Pulliainen. --- Lib/httplib.py | 7 +++++++ Lib/test/test_httplib.py | 7 +++++++ Misc/NEWS | 3 +++ 3 files changed, 17 insertions(+) diff --git a/Lib/httplib.py b/Lib/httplib.py index 680e8750413..8c2eab6dfef 100644 --- a/Lib/httplib.py +++ b/Lib/httplib.py @@ -211,6 +211,10 @@ # maximal amount of data to read at one time in _safe_read MAXAMOUNT = 1048576 +# maximum amount of headers accepted +_MAXHEADERS = 100 + + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -267,6 +271,8 @@ def readheaders(self): elif self.seekable: tell = self.fp.tell while True: + if len(hlist) > _MAXHEADERS: + raise HTTPException("got more than %d headers" % _MAXHEADERS) if tell: try: startofline = tell() @@ -1203,6 +1209,7 @@ def __init__(self, line): self.args = line, self.line = line + # for backwards compatibility error = HTTPException diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 730e8852a1d..b5f8a795437 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -152,6 +152,13 @@ def test_read_head(self): if resp.read() != "": self.fail("Did not expect response from HEAD request") + def test_too_many_headers(self): + headers = '\r\n'.join('Header%d: foo' % i for i in xrange(200)) + '\r\n' + text = ('HTTP/1.1 200 OK\r\n' + headers) + s = FakeSocket(text) + r = httplib.HTTPResponse(s) + self.assertRaises(httplib.HTTPException, r.begin) + def test_send_file(self): expected = 'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \ 'Accept-Encoding: identity\r\nContent-Length:' diff --git a/Misc/NEWS b/Misc/NEWS index 8a0db6e1ca7..0f31f1e2994 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins Library ------- +- Issue #16037: HTTPMessage.readheaders() raises an HTTPException when more + than 100 headers are read. Adapted from patch by Jyrki Pulliainen. + - Issue #16038: CVE-2013-1752: ftplib: Limit amount of data read by limiting the call to readline(). Original patch by MichaÅ‚ JastrzÄ™bski and Giampaolo Rodola. From adabc4cb1f03798e07570b41abcacb9236430039 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 29 Sep 2013 18:57:42 -0700 Subject: [PATCH 395/480] Code improvement. Review comment by Eric V. Smith --- Lib/SimpleHTTPServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/SimpleHTTPServer.py b/Lib/SimpleHTTPServer.py index 1996f4d29ba..1a172968658 100644 --- a/Lib/SimpleHTTPServer.py +++ b/Lib/SimpleHTTPServer.py @@ -150,7 +150,7 @@ def translate_path(self, path): path = path.split('?',1)[0] path = path.split('#',1)[0] # Don't forget explicit trailing slash when normalizing. Issue17324 - trailing_slash = True if path.rstrip().endswith('/') else False + trailing_slash = path.rstrip().endswith('/') path = posixpath.normpath(urllib.unquote(path)) words = path.split('/') words = filter(None, words) From 93b483650f9762afc3cb09563d3585035d3c96ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Mon, 30 Sep 2013 16:18:31 +0200 Subject: [PATCH 396/480] Add fake buildbottouch target. --- Makefile.pre.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 98711ae62bb..127328c91eb 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -796,6 +796,9 @@ buildbottest: all platform fi $(TESTPYTHON) -R $(TESTPROG) -uall -rwW $(TESTOPTS) +# Fake target in 2.7, real in 3.x. +buildbottouch: + QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ test_multibytecodec test_urllib2_localnet test_itertools \ test_multiprocessing test_mailbox test_socket test_poll \ From 24bd9bd373dd05bb09609a1135816f5767295c00 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 30 Sep 2013 21:35:44 +0200 Subject: [PATCH 397/480] Add a "skull and crossbones" to Py_AddPendingCall. --- Doc/c-api/init.rst | 55 +++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 6c58c5d2f05..7a6b0a6aee2 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -427,6 +427,9 @@ pointer. standard :mod:`zlib` and :mod:`hashlib` modules release the GIL when compressing or hashing data. + +.. _gilstate: + Non-Python created threads -------------------------- @@ -905,41 +908,43 @@ Asynchronous Notifications A mechanism is provided to make asynchronous notifications to the main interpreter thread. These notifications take the form of a function -pointer and a void argument. +pointer and a void pointer argument. -.. index:: single: setcheckinterval() (in module sys) - -Every check interval, when the global interpreter lock is released and -reacquired, Python will also call any such provided functions. This can be used -for example by asynchronous IO handlers. The notification can be scheduled from -a worker thread and the actual call than made at the earliest convenience by the -main thread where it has possession of the global interpreter lock and can -perform any Python API calls. .. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) .. index:: single: Py_AddPendingCall() - Post a notification to the Python main thread. If successful, *func* will be - called with the argument *arg* at the earliest convenience. *func* will be - called having the global interpreter lock held and can thus use the full - Python API and can take any action such as setting object attributes to - signal IO completion. It must return 0 on success, or -1 signalling an - exception. The notification function won't be interrupted to perform another - asynchronous notification recursively, but it can still be interrupted to - switch threads if the global interpreter lock is released, for example, if it - calls back into Python code. + Schedule a function to be called from the main interpreter thread. On + success, 0 is returned and *func* is queued for being called in the + main thread. On failure, -1 is returned without setting any exception. - This function returns 0 on success in which case the notification has been - scheduled. Otherwise, for example if the notification buffer is full, it - returns -1 without setting any exception. + When successfully queued, *func* will be *eventually* called from the + main interpreter thread with the argument *arg*. It will be called + asynchronously with respect to normally running Python code, but with + both these conditions met: - This function can be called on any thread, be it a Python thread or some - other system thread. If it is a Python thread, it doesn't matter if it holds - the global interpreter lock or not. + * on a :term:`bytecode` boundary; + * with the main thread holding the :term:`global interpreter lock` + (*func* can therefore use the full C API). - .. versionadded:: 2.7 + *func* must return 0 on success, or -1 on failure with an exception + set. *func* won't be interrupted to perform another asynchronous + notification recursively, but it can still be interrupted to switch + threads if the global interpreter lock is released. + This function doesn't need a current thread state to run, and it doesn't + need the global interpreter lock. + + .. warning:: + This is a low-level function, only useful for very special cases. + There is no guarantee that *func* will be called as quick as + possible. If the main thread is busy executing a system call, + *func* won't be called before the system call returns. This + function is generally **not** suitable for calling Python code from + arbitrary C threads. Instead, use the :ref:`PyGILState API`. + + .. versionadded:: 2.7 .. _profiling: From 537738c1e3e66376f25def084f1cbd54f95ae3be Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 30 Sep 2013 15:56:29 -0400 Subject: [PATCH 398/480] - Issue #16041: CVE-2013-1752: poplib: Limit maximum line lengths to 2048 to prevent readline() calls from consuming too much member. Patch by Jyrki Pulliainen. --- Lib/poplib.py | 14 +++++++- Lib/test/test_poplib.py | 71 +++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 17 ++++++---- 3 files changed, 95 insertions(+), 7 deletions(-) diff --git a/Lib/poplib.py b/Lib/poplib.py index e2b33ef10f4..3283eb603ae 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -32,6 +32,12 @@ class error_proto(Exception): pass LF = '\n' CRLF = CR+LF +# maximal line length when calling readline(). This is to prevent +# reading arbitrary lenght lines. RFC 1939 limits POP3 line length to +# 512 characters, including CRLF. We have selected 2048 just to be on +# the safe side. +_MAXLINE = 2048 + class POP3: @@ -103,7 +109,10 @@ def _putcmd(self, line): # Raise error_proto('-ERR EOF') if the connection is closed. def _getline(self): - line = self.file.readline() + line = self.file.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise error_proto('line too long') + if self._debugging > 1: print '*get*', repr(line) if not line: raise error_proto('-ERR EOF') octets = len(line) @@ -363,7 +372,10 @@ def _getline(self): line = "" renewline = re.compile(r'.*?\n') match = renewline.match(self.buffer) + while not match: + if len(self.buffer) > _MAXLINE: + raise error_proto('line too long') self._fillBuffer() match = renewline.match(self.buffer) line = match.group(0) diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 18ed224f7a9..a4107cef327 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -1,3 +1,4 @@ +import os import socket import threading import poplib @@ -21,6 +22,34 @@ def server(evt, serv): serv.close() evt.set() + +def evil_server(evt, serv, use_ssl=False): + serv.listen(5) + try: + conn, addr = serv.accept() + if use_ssl: + conn = ssl.wrap_socket( + conn, + server_side=True, + certfile=CERTFILE, + ) + except socket.timeout: + pass + else: + if use_ssl: + try: + conn.do_handshake() + except ssl.SSLError, err: + if err.args[0] not in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + raise + conn.send("+ Hola mundo" * 1000 + "\n") + conn.close() + finally: + serv.close() + evt.set() + + class GeneralTests(TestCase): def setUp(self): @@ -65,8 +94,50 @@ def testTimeoutValue(self): pop.sock.close() +class EvilServerTests(TestCase): + use_ssl = False + + def setUp(self): + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(3) + self.port = test_support.bind_port(self.sock) + threading.Thread( + target=evil_server, + args=(self.evt, self.sock, self.use_ssl)).start() + time.sleep(.1) + + def tearDown(self): + self.evt.wait() + + def testTooLongLines(self): + self.assertRaises(poplib.error_proto, poplib.POP3, + 'localhost', self.port, timeout=30) + + +SUPPORTS_SSL = False + +if hasattr(poplib, 'POP3_SSL'): + import ssl + + SUPPORTS_SSL = True + CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, + "keycert.pem") + + class EvilSSLServerTests(EvilServerTests): + use_ssl = True + + def testTooLongLines(self): + self.assertRaises(poplib.error_proto, poplib.POP3_SSL, + 'localhost', self.port) + + def test_main(verbose=None): test_support.run_unittest(GeneralTests) + test_support.run_unittest(EvilServerTests) + + if SUPPORTS_SSL: + test_support.run_unittest(EvilSSLServerTests) if __name__ == '__main__': test_main() diff --git a/Misc/NEWS b/Misc/NEWS index 0f31f1e2994..b9d7746e999 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,15 +13,22 @@ Core and Builtins Library ------- -- Issue #16037: HTTPMessage.readheaders() raises an HTTPException when more - than 100 headers are read. Adapted from patch by Jyrki Pulliainen. +- Issue #16042: CVE-2013-1752: smtplib: Limit amount of data read by + limiting the call to readline(). Original patch by Christian Heimes. + +- Issue #16041: CVE-2013-1752: poplib: Limit maximum line lengths to 2048 to + prevent readline() calls from consuming too much member. Patch by Jyrki + Pulliainen. + +- Issue #16039: CVE-2013-1752: Change use of readline in imaplib module to + limit line length. Patch by Emil Lind. - Issue #16038: CVE-2013-1752: ftplib: Limit amount of data read by limiting the call to readline(). Original patch by MichaÅ‚ JastrzÄ™bski and Giampaolo Rodola. -- Issue #16039: CVE-2013-1752: Change use of readline in imaplib module to - limit line length. Patch by Emil Lind. +- Issue #16037: HTTPMessage.readheaders() raises an HTTPException when more + than 100 headers are read. Adapted from patch by Jyrki Pulliainen. - Issue #14984: On POSIX systems, when netrc is called without a filename argument (and therefore is reading the user's $HOME/.netrc file), it now @@ -32,8 +39,6 @@ Library - Issue #16248: Disable code execution from the user's home directory by tkinter when the -E flag is passed to Python. Patch by Zachary Ware. -- Issue #16042: CVE-2013-1752: smtplib: Limit amount of data read by - limiting the call to readline(). Original patch by Christian Heimes. Extension Modules ----------------- From c0a8a9e2a0ef52be9e18b4ca21d51c019813ae63 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 30 Sep 2013 22:13:17 +0200 Subject: [PATCH 399/480] Issue #12641: Avoid passing "-mno-cygwin" to the mingw32 compiler, except when necessary. Patch by Oscar Benjamin. --- Lib/distutils/cygwinccompiler.py | 28 +++++++++++++++++++++------- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/Lib/distutils/cygwinccompiler.py b/Lib/distutils/cygwinccompiler.py index a1ee815c6cc..5d116876a33 100644 --- a/Lib/distutils/cygwinccompiler.py +++ b/Lib/distutils/cygwinccompiler.py @@ -319,13 +319,18 @@ def __init__ (self, else: entry_point = '' - self.set_executables(compiler='gcc -mno-cygwin -O -Wall', - compiler_so='gcc -mno-cygwin -mdll -O -Wall', - compiler_cxx='g++ -mno-cygwin -O -Wall', - linker_exe='gcc -mno-cygwin', - linker_so='%s -mno-cygwin %s %s' - % (self.linker_dll, shared_option, - entry_point)) + if self.gcc_version < '4' or is_cygwingcc(): + no_cygwin = ' -mno-cygwin' + else: + no_cygwin = '' + + self.set_executables(compiler='gcc%s -O -Wall' % no_cygwin, + compiler_so='gcc%s -mdll -O -Wall' % no_cygwin, + compiler_cxx='g++%s -O -Wall' % no_cygwin, + linker_exe='gcc%s' % no_cygwin, + linker_so='%s%s %s %s' + % (self.linker_dll, no_cygwin, + shared_option, entry_point)) # Maybe we should also append -mthreads, but then the finished # dlls need another dll (mingwm10.dll see Mingw32 docs) # (-mthreads: Support thread-safe exception handling on `Mingw32') @@ -447,3 +452,12 @@ def get_versions(): else: dllwrap_version = None return (gcc_version, ld_version, dllwrap_version) + +def is_cygwingcc(): + '''Try to determine if the gcc that would be used is from cygwin.''' + out = os.popen('gcc -dumpmachine', 'r') + out_string = out.read() + out.close() + # out_string is the target triplet cpu-vendor-os + # Cygwin's gcc sets the os to 'cygwin' + return out_string.strip().endswith('cygwin') diff --git a/Misc/ACKS b/Misc/ACKS index 6048ed8dd71..cbef1f00fcf 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -84,6 +84,7 @@ Ben Bell Thomas Bellman Alexander “Саша†Belopolsky David Benjamin +Oscar Benjamin Andrew Bennetts Andy Bensky Bennett Benson diff --git a/Misc/NEWS b/Misc/NEWS index af6ed7e96ca..54753cd9239 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #12641: Avoid passing "-mno-cygwin" to the mingw32 compiler, except + when necessary. Patch by Oscar Benjamin. + - Properly initialize all fields of a SSL object after allocation. - Issue #4366: Fix building extensions on all platforms when --enable-shared From 225faff1e18f1797611388c9c25e8e082b9ad308 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 30 Sep 2013 16:45:40 -0400 Subject: [PATCH 400/480] Fix typo in NEWS file. --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index b9d7746e999..7a14b10da6a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,7 +17,7 @@ Library limiting the call to readline(). Original patch by Christian Heimes. - Issue #16041: CVE-2013-1752: poplib: Limit maximum line lengths to 2048 to - prevent readline() calls from consuming too much member. Patch by Jyrki + prevent readline() calls from consuming too much memory. Patch by Jyrki Pulliainen. - Issue #16039: CVE-2013-1752: Change use of readline in imaplib module to From edca9556696a33d505d780f69d0933d9502a3098 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 30 Sep 2013 18:35:15 -0400 Subject: [PATCH 401/480] - Issue #16040: CVE-2013-1752: nntplib: Limit maximum line lengths to 2048 to prevent readline() calls from consuming too much memory. Patch by Jyrki Pulliainen. --- Lib/nntplib.py | 11 ++++++- Lib/test/test_nntplib.py | 65 ++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 4 +++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_nntplib.py diff --git a/Lib/nntplib.py b/Lib/nntplib.py index f519b06e55b..f5e0d29c7ce 100644 --- a/Lib/nntplib.py +++ b/Lib/nntplib.py @@ -37,6 +37,13 @@ "error_reply","error_temp","error_perm","error_proto", "error_data",] +# maximal line length when calling readline(). This is to prevent +# reading arbitrary lenght lines. RFC 3977 limits NNTP line length to +# 512 characters, including CRLF. We have selected 2048 just to be on +# the safe side. +_MAXLINE = 2048 + + # Exceptions raised when an error or invalid response is received class NNTPError(Exception): """Base class for all nntplib exceptions""" @@ -200,7 +207,9 @@ def putcmd(self, line): def getline(self): """Internal: return one line from the server, stripping CRLF. Raise EOFError if the connection is closed.""" - line = self.file.readline() + line = self.file.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise NNTPDataError('line too long') if self.debugging > 1: print '*get*', repr(line) if not line: raise EOFError diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py new file mode 100644 index 00000000000..31d41e621cd --- /dev/null +++ b/Lib/test/test_nntplib.py @@ -0,0 +1,65 @@ +import socket +import threading +import nntplib +import time + +from unittest import TestCase +from test import test_support + +HOST = test_support.HOST + + +def server(evt, serv, evil=False): + serv.listen(5) + try: + conn, addr = serv.accept() + except socket.timeout: + pass + else: + if evil: + conn.send("1 I'm too long response" * 3000 + "\n") + else: + conn.send("1 I'm OK response\n") + conn.close() + finally: + serv.close() + evt.set() + + +class BaseServerTest(TestCase): + def setUp(self): + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(3) + self.port = test_support.bind_port(self.sock) + threading.Thread( + target=server, + args=(self.evt, self.sock, self.evil)).start() + time.sleep(.1) + + def tearDown(self): + self.evt.wait() + + +class ServerTests(BaseServerTest): + evil = False + + def test_basic_connect(self): + nntp = nntplib.NNTP('localhost', self.port) + nntp.sock.close() + + +class EvilServerTests(BaseServerTest): + evil = True + + def test_too_long_line(self): + self.assertRaises(nntplib.NNTPDataError, + nntplib.NNTP, 'localhost', self.port) + + +def test_main(verbose=None): + test_support.run_unittest(EvilServerTests) + test_support.run_unittest(ServerTests) + +if __name__ == '__main__': + test_main() diff --git a/Misc/NEWS b/Misc/NEWS index 7a14b10da6a..dc1dd4b991f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -20,6 +20,10 @@ Library prevent readline() calls from consuming too much memory. Patch by Jyrki Pulliainen. +- Issue #16040: CVE-2013-1752: nntplib: Limit maximum line lengths to 2048 to + prevent readline() calls from consuming too much memory. Patch by Jyrki + Pulliainen. + - Issue #16039: CVE-2013-1752: Change use of readline in imaplib module to limit line length. Patch by Emil Lind. From dd9c8637cadff563492be788e759905caa40ca8b Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 30 Sep 2013 20:31:56 -0400 Subject: [PATCH 402/480] Regenerate pydoc_topics.py --- Lib/pydoc_topics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pydoc_topics.py b/Lib/pydoc_topics.py index 63558473fd4..18c8fda213d 100644 --- a/Lib/pydoc_topics.py +++ b/Lib/pydoc_topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue Apr 10 10:39:11 2012 +# Autogenerated by Sphinx on Mon Sep 30 20:24:25 2013 topics = {'assert': u'\nThe ``assert`` statement\n************************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, ``assert expression``, is equivalent to\n\n if __debug__:\n if not expression: raise AssertionError\n\nThe extended form, ``assert expression1, expression2``, is equivalent\nto\n\n if __debug__:\n if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that ``__debug__`` and ``AssertionError``\nrefer to the built-in variables with those names. In the current\nimplementation, the built-in variable ``__debug__`` is ``True`` under\nnormal circumstances, ``False`` when optimization is requested\n(command line option -O). The current code generator emits no code\nfor an assert statement when optimization is requested at compile\ntime. Note that it is unnecessary to include the source code for the\nexpression that failed in the error message; it will be displayed as\npart of the stack trace.\n\nAssignments to ``__debug__`` are illegal. The value for the built-in\nvariable is determined when the interpreter starts.\n', 'assignment': u'\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list is recursively defined as\nfollows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets. (This rule is relaxed as of\n Python 1.5; in earlier versions, the object had to be a tuple.\n Since strings are sequences, an assignment like ``a, b = "xy"`` is\n now legal as long as the string has the right length.)\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` statement in the\n current code block: the name is bound to the object in the current\n local namespace.\n\n * Otherwise: the name is bound to the object in the current global\n namespace.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield a plain integer. If it is negative, the\n sequence\'s length is added to it. The resulting value must be a\n nonnegative integer less than the sequence\'s length, and the\n sequence is asked to assign the assigned object to its item with\n that index. If the index is out of range, ``IndexError`` is raised\n (assignment to a subscripted sequence cannot add new items to a\n list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to (small) integers. If either\n bound is negative, the sequence\'s length is added to it. The\n resulting bounds are clipped to lie between zero and the sequence\'s\n length, inclusive. Finally, the sequence object is asked to replace\n the slice with the items of the assigned sequence. The length of\n the slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print x\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', 'atom-identifiers': u'\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name in front of the name, with leading underscores removed, and\na single underscore inserted in front of the class name. For example,\nthe identifier ``__spam`` occurring in a class named ``Ham`` will be\ntransformed to ``_Ham__spam``. This transformation is independent of\nthe syntactical context in which the identifier is used. If the\ntransformed name is extremely long (longer than 255 characters),\nimplementation defined truncation may happen. If the class name\nconsists only of underscores, no transformation is done.\n', From 5c9858480f8c1e980fbb8133017bc9d47659d84c Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 30 Sep 2013 20:34:29 -0400 Subject: [PATCH 403/480] Bump to 2.6.9rc1 --- Include/patchlevel.h | 8 ++++---- Lib/distutils/__init__.py | 2 +- Lib/idlelib/idlever.py | 2 +- Misc/NEWS | 2 +- Misc/RPM/python-2.6.spec | 2 +- README | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 36673ae8091..e45eb6efbd5 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -22,12 +22,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 2 #define PY_MINOR_VERSION 6 -#define PY_MICRO_VERSION 8 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL -#define PY_RELEASE_SERIAL 0 +#define PY_MICRO_VERSION 9 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA +#define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "2.6.8+" +#define PY_VERSION "2.6.9rc1" /*--end constants--*/ /* Subversion Revision number of this file (not of the repository) */ diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py index e5008c97170..fe33506f4cc 100644 --- a/Lib/distutils/__init__.py +++ b/Lib/distutils/__init__.py @@ -22,5 +22,5 @@ # #--start constants-- -__version__ = "2.6.8" +__version__ = "2.6.9rc1" #--end constants-- diff --git a/Lib/idlelib/idlever.py b/Lib/idlelib/idlever.py index 92525ae32a7..bed231baae2 100644 --- a/Lib/idlelib/idlever.py +++ b/Lib/idlelib/idlever.py @@ -1 +1 @@ -IDLE_VERSION = "2.6.8" +IDLE_VERSION = "2.6.9rc1" diff --git a/Misc/NEWS b/Misc/NEWS index dc1dd4b991f..671c9890a5b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -5,7 +5,7 @@ Python News What's New in Python 2.6.9 rc 1? ================================ -*Release date: XXXX-XX-XX* +*Release date: 2013-09-30* Core and Builtins ----------------- diff --git a/Misc/RPM/python-2.6.spec b/Misc/RPM/python-2.6.spec index e80b952b969..e2de7e36ca2 100644 --- a/Misc/RPM/python-2.6.spec +++ b/Misc/RPM/python-2.6.spec @@ -39,7 +39,7 @@ %define name python #--start constants-- -%define version 2.6.8 +%define version 2.6.9rc1 %define libvers 2.6 #--end constants-- %define release 1pydotorg diff --git a/README b/README index 9cda7241106..57c6550d1d3 100644 --- a/README +++ b/README @@ -1,8 +1,8 @@ -This is Python version 2.6.8 +This is Python version 2.6.9 ============================ Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, - 2011, 2012 + 2011, 2012, 2013 Python Software Foundation. All rights reserved. From 3d566207acabf655a47e0bde171a92c620edf75e Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 30 Sep 2013 20:37:45 -0400 Subject: [PATCH 404/480] Bump copyright years. --- Doc/README.txt | 2 +- Doc/copyright.rst | 2 +- Doc/license.rst | 2 ++ LICENSE | 1 + PC/python_nt.rc | 2 +- Python/getcopyright.c | 2 +- README | 4 ++-- 7 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Doc/README.txt b/Doc/README.txt index 22da427c101..913e44d2971 100644 --- a/Doc/README.txt +++ b/Doc/README.txt @@ -127,7 +127,7 @@ The Python source is copyrighted, but you can freely use and copy it as long as you don't change or remove the copyright notice: ---------------------------------------------------------------------- -Copyright (c) 2000-2012 Python Software Foundation. +Copyright (c) 2000-2013 Python Software Foundation. All rights reserved. Copyright (c) 2000 BeOpen.com. diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 9a245c8b723..94f777c4ef6 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2012 Python Software Foundation. All rights reserved. +Copyright © 2001-2013 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/license.rst b/Doc/license.rst index a38a167cdeb..d36cb4479b3 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -110,6 +110,8 @@ been GPL-compatible; the table below summarizes the various releases. +----------------+--------------+-----------+------------+-----------------+ | 2.6.8 | 2.6.7 | 2012 | PSF | yes | +----------------+--------------+-----------+------------+-----------------+ +| 2.6.9 | 2.6.8 | 2013 | PSF | yes | ++----------------+--------------+-----------+------------+-----------------+ .. note:: diff --git a/LICENSE b/LICENSE index 4e7becb7e9f..a52282251a1 100644 --- a/LICENSE +++ b/LICENSE @@ -66,6 +66,7 @@ the various releases. 2.6.6 2.6.5 2010 PSF yes 2.6.7 2.6.6 2011 PSF yes 2.6.8 2.6.7 2012 PSF yes + 2.6.9 2.6.8 2013 PSF yes Footnotes: diff --git a/PC/python_nt.rc b/PC/python_nt.rc index 5295610ae17..f31c8368782 100644 --- a/PC/python_nt.rc +++ b/PC/python_nt.rc @@ -61,7 +61,7 @@ BEGIN VALUE "FileDescription", "Python Core\0" VALUE "FileVersion", PYTHON_VERSION VALUE "InternalName", "Python DLL\0" - VALUE "LegalCopyright", "Copyright © 2001-2012 Python Software Foundation. Copyright © 2000 BeOpen.com. Copyright © 1995-2001 CNRI. Copyright © 1991-1995 SMC.\0" + VALUE "LegalCopyright", "Copyright © 2001-2013 Python Software Foundation. Copyright © 2000 BeOpen.com. Copyright © 1995-2001 CNRI. Copyright © 1991-1995 SMC.\0" VALUE "OriginalFilename", PYTHON_DLL_NAME "\0" VALUE "ProductName", "Python\0" VALUE "ProductVersion", PYTHON_VERSION diff --git a/Python/getcopyright.c b/Python/getcopyright.c index d91a879ae38..4f2009deea9 100644 --- a/Python/getcopyright.c +++ b/Python/getcopyright.c @@ -4,7 +4,7 @@ static char cprt[] = "\ -Copyright (c) 2001-2012 Python Software Foundation.\n\ +Copyright (c) 2001-2013 Python Software Foundation.\n\ All Rights Reserved.\n\ \n\ Copyright (c) 2000 BeOpen.com.\n\ diff --git a/README b/README index 57c6550d1d3..044d1b3d2b8 100644 --- a/README +++ b/README @@ -1,5 +1,5 @@ -This is Python version 2.6.9 -============================ +This is Python version 2.6.9rc1 +=============================== Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 From 5e5b0537bc8e02b31320a0b6f346983eca4c21dd Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 30 Sep 2013 20:37:58 -0400 Subject: [PATCH 405/480] Added tag v2.6.9rc1 for changeset a0025037f11a From 450da6f46d3088ac75f2973542cb708e237c6976 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 30 Sep 2013 21:49:31 -0400 Subject: [PATCH 406/480] post release bump --- Include/patchlevel.h | 2 +- Misc/NEWS | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index e45eb6efbd5..a557ce79afb 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -27,7 +27,7 @@ #define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "2.6.9rc1" +#define PY_VERSION "2.6.9rc1+" /*--end constants--*/ /* Subversion Revision number of this file (not of the repository) */ diff --git a/Misc/NEWS b/Misc/NEWS index 671c9890a5b..5ee935a4f7f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,18 @@ Python News +++++++++++ +What's New in Python 2.6.9? +=========================== + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +Library +------- + + What's New in Python 2.6.9 rc 1? ================================ From 4d5d60747c2f894785f21b23898265b7bf0fb05b Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Mon, 30 Sep 2013 22:10:44 -0700 Subject: [PATCH 407/480] Minor doc fix in urlparse.rst --- Doc/library/urlparse.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/urlparse.rst b/Doc/library/urlparse.rst index efd112d1758..1bc361d8b26 100644 --- a/Doc/library/urlparse.rst +++ b/Doc/library/urlparse.rst @@ -72,7 +72,7 @@ The :mod:`urlparse` module defines the following functions: ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') >>> urlparse('www.cwi.nl/%7Eguido/Python.html') - ParseResult(scheme='', netloc='', path='www.cwi.nl:80/%7Eguido/Python.html', + ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', params='', query='', fragment='') >>> urlparse('help/Python.html') ParseResult(scheme='', netloc='', path='help/Python.html', params='', From 8eb18ecc938eaabfedb474d1dd7eb84490d98cad Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 1 Oct 2013 11:38:38 -0400 Subject: [PATCH 408/480] Typo caught by : Arfrever Frehtes Taifersar Arahesis. --- Lib/nntplib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/nntplib.py b/Lib/nntplib.py index f5e0d29c7ce..32962b97596 100644 --- a/Lib/nntplib.py +++ b/Lib/nntplib.py @@ -38,7 +38,7 @@ "error_data",] # maximal line length when calling readline(). This is to prevent -# reading arbitrary lenght lines. RFC 3977 limits NNTP line length to +# reading arbitrary length lines. RFC 3977 limits NNTP line length to # 512 characters, including CRLF. We have selected 2048 just to be on # the safe side. _MAXLINE = 2048 From 4fbc489b82fbd664838e8fccc202cb13277132ce Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 2 Oct 2013 11:40:26 +0300 Subject: [PATCH 409/480] Issue #19137: The pprint module now correctly formats empty set and frozenset and instances of set and frozenset subclasses. --- Lib/pprint.py | 23 ++++----- Lib/test/test_pprint.py | 104 ++++++++++++++++++++++++++++++---------- Misc/NEWS | 3 ++ 3 files changed, 91 insertions(+), 39 deletions(-) diff --git a/Lib/pprint.py b/Lib/pprint.py index 330099dfea6..77f2a566f35 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -185,25 +185,18 @@ def _format(self, object, stream, indent, allowance, context, level): if issubclass(typ, list): write('[') endchar = ']' - elif issubclass(typ, set): - if not length: - write('set()') - return - write('set([') - endchar = '])' - object = _sorted(object) - indent += 4 - elif issubclass(typ, frozenset): + elif issubclass(typ, tuple): + write('(') + endchar = ')' + else: if not length: - write('frozenset()') + write(rep) return - write('frozenset([') + write(typ.__name__) + write('([') endchar = '])' + indent += len(typ.__name__) + 1 object = _sorted(object) - indent += 10 - else: - write('(') - endchar = ')' if self._indent_per_level > 1 and sepLines: write((self._indent_per_level - 1) * ' ') if length: diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index e0137dfc59f..50493f6d1a8 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -24,6 +24,20 @@ class tuple3(tuple): def __repr__(self): return tuple.__repr__(self) +class set2(set): + pass + +class set3(set): + def __repr__(self): + return set.__repr__(self) + +class frozenset2(frozenset): + pass + +class frozenset3(frozenset): + def __repr__(self): + return frozenset.__repr__(self) + class dict2(dict): pass @@ -114,22 +128,24 @@ def test_same_as_repr(self): for simple in (0, 0L, 0+0j, 0.0, "", uni(""), (), tuple2(), tuple3(), [], list2(), list3(), + set(), set2(), set3(), + frozenset(), frozenset2(), frozenset3(), {}, dict2(), dict3(), self.assertTrue, pprint, -6, -6L, -6-6j, -1.5, "x", uni("x"), (3,), [3], {3: 6}, (1,2), [3,4], {5: 6}, tuple2((1,2)), tuple3((1,2)), tuple3(range(100)), [3,4], list2([3,4]), list3([3,4]), list3(range(100)), + set({7}), set2({7}), set3({7}), + frozenset({8}), frozenset2({8}), frozenset3({8}), dict2({5: 6}), dict3({5: 6}), range(10, -11, -1) ): native = repr(simple) - for function in "pformat", "saferepr": - f = getattr(pprint, function) - got = f(simple) - self.assertEqual(native, got, - "expected %s got %s from pprint.%s" % - (native, got, function)) + self.assertEqual(pprint.pformat(simple), native) + self.assertEqual(pprint.pformat(simple, width=1, indent=0) + .replace('\n', ' '), native) + self.assertEqual(pprint.saferepr(simple), native) def test_basic_line_wrap(self): # verify basic line-wrapping operation @@ -205,19 +221,59 @@ def test_subclassing(self): self.assertEqual(DottedPrettyPrinter().pformat(o), exp) def test_set_reprs(self): - self.assertEqual(pprint.pformat(set()), 'set()') + self.assertEqual(pprint.pformat(set()), 'set([])') self.assertEqual(pprint.pformat(set(range(3))), 'set([0, 1, 2])') - self.assertEqual(pprint.pformat(frozenset()), 'frozenset()') - self.assertEqual(pprint.pformat(frozenset(range(3))), 'frozenset([0, 1, 2])') + self.assertEqual(pprint.pformat(set(range(7)), width=20), '''\ +set([0, + 1, + 2, + 3, + 4, + 5, + 6])''') + self.assertEqual(pprint.pformat(set2(range(7)), width=20), '''\ +set2([0, + 1, + 2, + 3, + 4, + 5, + 6])''') + self.assertEqual(pprint.pformat(set3(range(7)), width=20), + 'set3([0, 1, 2, 3, 4, 5, 6])') + + self.assertEqual(pprint.pformat(frozenset()), 'frozenset([])') + self.assertEqual(pprint.pformat(frozenset(range(3))), + 'frozenset([0, 1, 2])') + self.assertEqual(pprint.pformat(frozenset(range(7)), width=20), '''\ +frozenset([0, + 1, + 2, + 3, + 4, + 5, + 6])''') + self.assertEqual(pprint.pformat(frozenset2(range(7)), width=20), '''\ +frozenset2([0, + 1, + 2, + 3, + 4, + 5, + 6])''') + self.assertEqual(pprint.pformat(frozenset3(range(7)), width=20), + 'frozenset3([0, 1, 2, 3, 4, 5, 6])') + + def test_set_of_sets_reprs(self): cube_repr_tgt = """\ {frozenset([]): frozenset([frozenset([2]), frozenset([0]), frozenset([1])]), - frozenset([0]): frozenset([frozenset(), + frozenset([0]): frozenset([frozenset([]), frozenset([0, 2]), frozenset([0, 1])]), - frozenset([1]): frozenset([frozenset(), + frozenset([1]): frozenset([frozenset([]), frozenset([1, 2]), frozenset([0, 1])]), - frozenset([2]): frozenset([frozenset(), + frozenset([2]): frozenset([frozenset([]), frozenset([1, 2]), frozenset([0, 2])]), frozenset([1, 2]): frozenset([frozenset([2]), @@ -243,7 +299,7 @@ def test_set_reprs(self): frozenset([frozenset([0]), frozenset([0, 1])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([0])]), frozenset([frozenset([2]), frozenset([0, @@ -259,7 +315,7 @@ def test_set_reprs(self): frozenset([frozenset([1]), frozenset([1, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([1])])]), frozenset([frozenset([1, 2]), frozenset([1])]): frozenset([frozenset([frozenset([1, 2]), @@ -269,7 +325,7 @@ def test_set_reprs(self): frozenset([frozenset([2]), frozenset([1, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([1])]), frozenset([frozenset([1]), frozenset([0, @@ -285,7 +341,7 @@ def test_set_reprs(self): frozenset([frozenset([2]), frozenset([0, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([2])])]), frozenset([frozenset([]), frozenset([0])]): frozenset([frozenset([frozenset([0]), frozenset([0, @@ -293,16 +349,16 @@ def test_set_reprs(self): frozenset([frozenset([0]), frozenset([0, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([1])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([2])])]), - frozenset([frozenset([]), frozenset([1])]): frozenset([frozenset([frozenset(), + frozenset([frozenset([]), frozenset([1])]): frozenset([frozenset([frozenset([]), frozenset([0])]), frozenset([frozenset([1]), frozenset([1, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([2])]), frozenset([frozenset([1]), frozenset([0, @@ -310,9 +366,9 @@ def test_set_reprs(self): frozenset([frozenset([2]), frozenset([])]): frozenset([frozenset([frozenset([2]), frozenset([1, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([0])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([1])]), frozenset([frozenset([2]), frozenset([0, @@ -333,7 +389,7 @@ def test_set_reprs(self): frozenset([frozenset([1]), frozenset([0, 1])])]), - frozenset([frozenset([0]), frozenset([0, 1])]): frozenset([frozenset([frozenset(), + frozenset([frozenset([0]), frozenset([0, 1])]): frozenset([frozenset([frozenset([]), frozenset([0])]), frozenset([frozenset([0, 1]), @@ -357,7 +413,7 @@ def test_set_reprs(self): frozenset([frozenset([0]), frozenset([0, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([2])])]), frozenset([frozenset([0, 1, 2]), frozenset([0, 2])]): frozenset([frozenset([frozenset([1, 2]), diff --git a/Misc/NEWS b/Misc/NEWS index 191a87e1001..feb3f080b09 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #19137: The pprint module now correctly formats empty set and frozenset + and instances of set and frozenset subclasses. + - Issue #16040: CVE-2013-1752: nntplib: Limit maximum line lengths to 2048 to prevent readline() calls from consuming too much memory. Patch by Jyrki Pulliainen. From 8cb97894c38daffd93e6c047c037b93b7279dc29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Wed, 2 Oct 2013 17:03:05 +0200 Subject: [PATCH 410/480] Drop buildbottouch again; the master is using "make touch" now. --- Makefile.pre.in | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 127328c91eb..98711ae62bb 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -796,9 +796,6 @@ buildbottest: all platform fi $(TESTPYTHON) -R $(TESTPROG) -uall -rwW $(TESTOPTS) -# Fake target in 2.7, real in 3.x. -buildbottouch: - QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ test_multibytecodec test_urllib2_localnet test_itertools \ test_multiprocessing test_mailbox test_socket test_poll \ From 493950d9531a62ff2c5ff82724b6731949317bb9 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Wed, 2 Oct 2013 12:20:18 -0700 Subject: [PATCH 411/480] Issue #19147: Fix docstring for fcntl.flock to refer to correct man section. --- Modules/fcntlmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 997867a136e..cd5c98f1813 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -299,7 +299,7 @@ PyDoc_STRVAR(flock_doc, "flock(fd, operation)\n\ \n\ Perform the lock operation op on file descriptor fd. See the Unix \n\ -manual page for flock(3) for details. (On some systems, this function is\n\ +manual page for flock(2) for details. (On some systems, this function is\n\ emulated using fcntl().)"); From eca0ede7bf1c7a2b6053645cb1206e2d6b6b27d4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 3 Oct 2013 12:08:22 +0300 Subject: [PATCH 412/480] Issue #18037: 2to3 now escapes '\u' and '\U' in native strings. --- Lib/lib2to3/fixes/fix_unicode.py | 32 +++++++++++++++++++++------ Lib/lib2to3/tests/test_fixers.py | 37 ++++++++++++++++++++++++++++++++ Misc/NEWS | 2 ++ 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/Lib/lib2to3/fixes/fix_unicode.py b/Lib/lib2to3/fixes/fix_unicode.py index 6c89576540c..922486b5d85 100644 --- a/Lib/lib2to3/fixes/fix_unicode.py +++ b/Lib/lib2to3/fixes/fix_unicode.py @@ -1,25 +1,43 @@ -"""Fixer that changes unicode to str, unichr to chr, and u"..." into "...". +r"""Fixer for unicode. + +* Changes unicode to str and unichr to chr. + +* If "...\u..." is not unicode literal change it into "...\\u...". + +* Change u"..." into "...". """ -import re from ..pgen2 import token from .. import fixer_base _mapping = {u"unichr" : u"chr", u"unicode" : u"str"} -_literal_re = re.compile(ur"[uU][rR]?[\'\"]") class FixUnicode(fixer_base.BaseFix): BM_compatible = True PATTERN = "STRING | 'unicode' | 'unichr'" + def start_tree(self, tree, filename): + super(FixUnicode, self).start_tree(tree, filename) + self.unicode_literals = 'unicode_literals' in tree.future_features + def transform(self, node, results): if node.type == token.NAME: new = node.clone() new.value = _mapping[node.value] return new elif node.type == token.STRING: - if _literal_re.match(node.value): - new = node.clone() - new.value = new.value[1:] - return new + val = node.value + if (not self.unicode_literals and val[0] in u'rR\'"' and + u'\\' in val): + val = ur'\\'.join([ + v.replace(u'\\u', ur'\\u').replace(u'\\U', ur'\\U') + for v in val.split(ur'\\') + ]) + if val[0] in u'uU': + val = val[1:] + if val == node.value: + return node + new = node.clone() + new.value = val + return new diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index 1817208c7c8..5f283a83598 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -2824,6 +2824,43 @@ def test_unicode_literal_3(self): a = """R'''x''' """ self.check(b, a) + def test_native_literal_escape_u(self): + b = """'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = """'\\\\\\\\u20ac\\\\U0001d121\\\\u20ac'""" + self.check(b, a) + + b = """r'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = """r'\\\\\\\\u20ac\\\\U0001d121\\\\u20ac'""" + self.check(b, a) + + def test_bytes_literal_escape_u(self): + b = """b'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = """b'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + + b = """br'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = """br'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + + def test_unicode_literal_escape_u(self): + b = """u'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = """'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + + b = """ur'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = """r'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + + def test_native_unicode_literal_escape_u(self): + f = 'from __future__ import unicode_literals\n' + b = f + """'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = f + """'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + + b = f + """r'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = f + """r'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + class Test_callable(FixerTestCase): fixer = "callable" diff --git a/Misc/NEWS b/Misc/NEWS index feb3f080b09..9aad84fc177 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,8 @@ Core and Builtins Library ------- +- Issue #18037: 2to3 now escapes '\u' and '\U' in native strings. + - Issue #19137: The pprint module now correctly formats empty set and frozenset and instances of set and frozenset subclasses. From e321c8a8f4d68f47db33339463099c3f327fafcc Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 4 Oct 2013 10:55:15 -0400 Subject: [PATCH 413/480] dict is also allowed (closes #19163) --- Doc/c-api/string.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/string.rst b/Doc/c-api/string.rst index ecf70504bf0..32dc274a9e1 100644 --- a/Doc/c-api/string.rst +++ b/Doc/c-api/string.rst @@ -241,7 +241,7 @@ called with a non-string parameter. .. c:function:: PyObject* PyString_Format(PyObject *format, PyObject *args) Return a new string object from *format* and *args*. Analogous to ``format % - args``. The *args* argument must be a tuple. + args``. The *args* argument must be a tuple or dict. .. c:function:: void PyString_InternInPlace(PyObject **string) From cb31bbe3183d1e05de17ec5252199cff70e7a2d6 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 5 Oct 2013 03:01:37 +0300 Subject: [PATCH 414/480] #19166: use an unused var in a test. Patch by Vajrasky Kok. --- Lib/test/test_dict.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 0840f59799b..a5685b98eb7 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -33,9 +33,11 @@ def test_keys(self): self.assertEqual(d.keys(), []) d = {'a': 1, 'b': 2} k = d.keys() + self.assertEqual(set(k), {'a', 'b'}) + self.assertIn('a', k) + self.assertIn('b', k) self.assertTrue(d.has_key('a')) self.assertTrue(d.has_key('b')) - self.assertRaises(TypeError, d.keys, None) def test_values(self): From 4a88763f2ed93f66d5368678cac367ad64092133 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 6 Oct 2013 00:39:18 +0300 Subject: [PATCH 415/480] #19068: use imperative mood in complex object docstrings. Patch by Marco Buttu. --- Objects/complexobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 677ac0ebe5a..5ee0c15e3d7 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -885,7 +885,7 @@ complex_conjugate(PyObject *self) PyDoc_STRVAR(complex_conjugate_doc, "complex.conjugate() -> complex\n" "\n" -"Returns the complex conjugate of its argument. (3-4j).conjugate() == 3+4j."); +"Return the complex conjugate of its argument. (3-4j).conjugate() == 3+4j."); static PyObject * complex_getnewargs(PyComplexObject *v) @@ -897,7 +897,7 @@ complex_getnewargs(PyComplexObject *v) PyDoc_STRVAR(complex__format__doc, "complex.__format__() -> str\n" "\n" -"Converts to a string according to format_spec."); +"Convert to a string according to format_spec."); static PyObject * complex__format__(PyObject* self, PyObject* args) From 29e2d39cf9e9987ca1bcca0a8d09cc2c9e5c1e0f Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 6 Oct 2013 00:44:32 +0300 Subject: [PATCH 416/480] #19069: use imperative mood in float object docstrings. Patch by Marco Buttu. --- Objects/floatobject.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index ba867ef149f..2bec0fbc6ff 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1783,9 +1783,9 @@ float_as_integer_ratio(PyObject *v, PyObject *unused) PyDoc_STRVAR(float_as_integer_ratio_doc, "float.as_integer_ratio() -> (int, int)\n" "\n" -"Returns a pair of integers, whose ratio is exactly equal to the original\n" +"Return a pair of integers, whose ratio is exactly equal to the original\n" "float and with a positive denominator.\n" -"Raises OverflowError on infinities and a ValueError on NaNs.\n" +"Raise OverflowError on infinities and a ValueError on NaNs.\n" "\n" ">>> (10.0).as_integer_ratio()\n" "(10, 1)\n" @@ -1970,7 +1970,7 @@ PyDoc_STRVAR(float_setformat_doc, "'IEEE, big-endian' or 'IEEE, little-endian', and in addition can only be\n" "one of the latter two if it appears to match the underlying C reality.\n" "\n" -"Overrides the automatic determination of C-level floating point type.\n" +"Override the automatic determination of C-level floating point type.\n" "This affects how floats are converted to and from binary strings."); static PyObject * @@ -2017,9 +2017,9 @@ PyDoc_STRVAR(float__format__doc, static PyMethodDef float_methods[] = { {"conjugate", (PyCFunction)float_float, METH_NOARGS, - "Returns self, the complex conjugate of any float."}, + "Return self, the complex conjugate of any float."}, {"__trunc__", (PyCFunction)float_trunc, METH_NOARGS, - "Returns the Integral closest to x between 0 and x."}, + "Return the Integral closest to x between 0 and x."}, {"as_integer_ratio", (PyCFunction)float_as_integer_ratio, METH_NOARGS, float_as_integer_ratio_doc}, {"fromhex", (PyCFunction)float_fromhex, @@ -2027,14 +2027,14 @@ static PyMethodDef float_methods[] = { {"hex", (PyCFunction)float_hex, METH_NOARGS, float_hex_doc}, {"is_integer", (PyCFunction)float_is_integer, METH_NOARGS, - "Returns True if the float is an integer."}, + "Return True if the float is an integer."}, #if 0 {"is_inf", (PyCFunction)float_is_inf, METH_NOARGS, - "Returns True if the float is positive or negative infinite."}, + "Return True if the float is positive or negative infinite."}, {"is_finite", (PyCFunction)float_is_finite, METH_NOARGS, - "Returns True if the float is finite, neither infinite nor NaN."}, + "Return True if the float is finite, neither infinite nor NaN."}, {"is_nan", (PyCFunction)float_is_nan, METH_NOARGS, - "Returns True if the float is not a number (NaN)."}, + "Return True if the float is not a number (NaN)."}, #endif {"__getnewargs__", (PyCFunction)float_getnewargs, METH_NOARGS}, {"__getformat__", (PyCFunction)float_getformat, From 74bd7980364615a39f81a39c36e8f876aa05a444 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 6 Oct 2013 01:02:09 +0300 Subject: [PATCH 417/480] #19074: mention PySide in the GUI FAQs. --- Doc/faq/gui.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index bb544202d9d..9667dcb7997 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -47,13 +47,15 @@ well as in freeware or shareware. Qt --- -There are bindings available for the Qt toolkit (`PyQt -`_) and for KDE (`PyKDE `__). If -you're writing open source software, you don't need to pay for PyQt, but if you -want to write proprietary applications, you must buy a PyQt license from -`Riverbank Computing `_ and (up to Qt 4.4; -Qt 4.5 upwards is licensed under the LGPL license) a Qt license from `Trolltech -`_. +There are bindings available for the Qt toolkit (using either `PyQt +`_ or `PySide +`_) and for KDE (`PyKDE http://www.riverbankcomputing.co.uk/software/pykde/intro>`__). +PyQt is currently more mature than PySide, but you must buy a PyQt license from +`Riverbank Computing `_ +if you want to write proprietary applications. PySide is free for all applications. + +Qt 4.5 upwards is licensed under the LGPL license; also, commercial licenses +are available from `Nokia `_. Gtk+ ---- From 9895e08af4aefdf6773617acf2f03d60f1a311b4 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sat, 5 Oct 2013 16:53:52 -0500 Subject: [PATCH 418/480] Issue #19171: speed some cases of 3-argument long pow(). Reduce the base by the modulus when the base is larger than the modulus. This can unboundedly speed the "startup costs" of doing modular exponentiation, particularly in cases where the base is much larger than the modulus. Original patch by Armin Rigo, inspired by https://github.com/pyca/ed25519. (grafted from f34c59494420765b013136ca93f63b716d9f1d30) --- Objects/longobject.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index fb740dc4d6c..405be2e045a 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3474,10 +3474,16 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) goto Done; } - /* if base < 0: - base = base % modulus - Having the base positive just makes things easier. */ - if (Py_SIZE(a) < 0) { + /* Reduce base by modulus in some cases: + 1. If base < 0. Forcing the base non-negative makes things easier. + 2. If base is obviously larger than the modulus. The "small + exponent" case later can multiply directly by base repeatedly, + while the "large exponent" case multiplies directly by base 31 + times. It can be unboundedly faster to multiply by + base % modulus instead. + We could _always_ do this reduction, but l_divmod() isn't cheap, + so we only do it when it buys something. */ + if (Py_SIZE(a) < 0 || Py_SIZE(a) > Py_SIZE(c)) { if (l_divmod(a, c, NULL, &temp) < 0) goto Error; Py_DECREF(a); From d493e52b6dd093b67407a90c4522b2ff17efb490 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 5 Oct 2013 21:34:48 -0700 Subject: [PATCH 419/480] Issue 14927: Minor clean-up of function parameters in random(). --- Lib/random.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/Lib/random.py b/Lib/random.py index 6fa1c050889..1a3a13ee862 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -170,29 +170,28 @@ def __reduce__(self): ## -------------------- integer methods ------------------- - def randrange(self, start, stop=None, step=1, int=int, default=None, - maxwidth=1L< 0: - if istart >= maxwidth: + if istart >= _maxwidth: return self._randbelow(istart) - return int(self.random() * istart) + return _int(self.random() * istart) raise ValueError, "empty range for randrange()" # stop argument supplied. - istop = int(stop) + istop = _int(stop) if istop != stop: raise ValueError, "non-integer stop for randrange()" width = istop - istart @@ -210,14 +209,14 @@ def randrange(self, start, stop=None, step=1, int=int, default=None, # a long, but we're supposed to return an int (for backward # compatibility). - if width >= maxwidth: - return int(istart + self._randbelow(width)) - return int(istart + int(self.random()*width)) + if width >= _maxwidth: + return _int(istart + self._randbelow(width)) + return _int(istart + _int(self.random()*width)) if step == 1: raise ValueError, "empty range for randrange() (%d,%d, %d)" % (istart, istop, width) # Non-unit step argument supplied. - istep = int(step) + istep = _int(step) if istep != step: raise ValueError, "non-integer step for randrange()" if istep > 0: @@ -230,9 +229,9 @@ def randrange(self, start, stop=None, step=1, int=int, default=None, if n <= 0: raise ValueError, "empty range for randrange()" - if n >= maxwidth: + if n >= _maxwidth: return istart + istep*self._randbelow(n) - return istart + istep*int(self.random() * n) + return istart + istep*_int(self.random() * n) def randint(self, a, b): """Return random integer in range [a, b], including both end points. @@ -240,7 +239,7 @@ def randint(self, a, b): return self.randrange(a, b+1) - def _randbelow(self, n, _log=_log, int=int, _maxwidth=1L< n-1 > 2**(k-2) + k = _int(1.00001 + _log(n-1, 2.0)) # 2**k > n-1 > 2**(k-2) r = getrandbits(k) while r >= n: r = getrandbits(k) @@ -265,7 +264,7 @@ def _randbelow(self, n, _log=_log, int=int, _maxwidth=1L<= _maxwidth: _warn("Underlying random() generator does not supply \n" "enough bits to choose from a population range this large") - return int(self.random() * n) + return _int(self.random() * n) ## -------------------- sequence methods ------------------- @@ -273,20 +272,20 @@ def choice(self, seq): """Choose a random element from a non-empty sequence.""" return seq[int(self.random() * len(seq))] # raises IndexError if seq is empty - def shuffle(self, x, random=None, int=int): + def shuffle(self, x, random=None): """x, random=random.random -> shuffle list x in place; return None. Optional arg random is a 0-argument function returning a random float in [0.0, 1.0); by default, the standard random.random. - Do not supply the 'int' argument. """ if random is None: random = self.random + _int = int for i in reversed(xrange(1, len(x))): # pick an element in x[:i+1] with which to exchange x[i] - j = int(random() * (i+1)) + j = _int(random() * (i+1)) x[i], x[j] = x[j], x[i] def sample(self, population, k): From b1c1191cde7a56e1dfe76df4c0a1aaf855f93ff0 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 09:17:43 +0200 Subject: [PATCH 420/480] Fix "customer timer" -> "custom timer" (reported by Kirk Strauser on docs@) --- Doc/library/profile.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 00abef2873e..0fb14892ff1 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -625,8 +625,8 @@ your results will "less often" show up as negative in profile statistics. .. _profile-timers: -Using a customer timer -====================== +Using a custom timer +==================== If you want to change how current time is determined (for example, to force use of wall-clock time or elapsed process time), pass the timing function you want From c0b735960cd0992e818576114ba8e1c8d08066ab Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 09:23:03 +0200 Subject: [PATCH 421/480] Fix: Element.text is an attribute, not a method (report by Cameron Laird on docs@) --- Doc/library/xml.etree.elementtree.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index fee56ff3fe9..24e50846c8f 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -158,7 +158,7 @@ the sub-tree below it (its children, their children, and so on). For example, :meth:`Element.findall` finds only elements with a tag which are direct children of the current element. :meth:`Element.find` finds the *first* child -with a particular tag, and :meth:`Element.text` accesses the element's text +with a particular tag, and :attr:`Element.text` accesses the element's text content. :meth:`Element.get` accesses the element's attributes:: >>> for country in root.findall('country'): From 1e7e124c05ca389f76f4fe445c1f8b492f0f0013 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 09:32:03 +0200 Subject: [PATCH 422/480] Remove dead link to effbot Tkinter introduction material (reported by Ralf Ganswindt from docs@) --- Doc/library/tkinter.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index dfb4da6de39..cf8eba06c27 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -41,9 +41,6 @@ is maintained at ActiveState.) `Modern Tkinter for Busy Python Developers `_ Book by Mark Rozerman about building attractive and modern graphical user interfaces with Python and Tkinter. - `An Introduction to Tkinter `_ - Fredrik Lundh's on-line reference material. - `Python and Tkinter Programming `_ The book by John Grayson (ISBN 1-884777-81-3). From 8eac5a9ff7340a476e2abaf40856037aeb20dd63 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 09:47:45 +0200 Subject: [PATCH 423/480] Fix markup. --- Doc/faq/gui.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 9667dcb7997..b34be2dcd6e 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -49,7 +49,7 @@ Qt There are bindings available for the Qt toolkit (using either `PyQt `_ or `PySide -`_) and for KDE (`PyKDE http://www.riverbankcomputing.co.uk/software/pykde/intro>`__). +`_) and for KDE (`PyKDE `_). PyQt is currently more mature than PySide, but you must buy a PyQt license from `Riverbank Computing `_ if you want to write proprietary applications. PySide is free for all applications. From 7dad23e7c1b467150180d8a96526e229be52868d Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 09:48:26 +0200 Subject: [PATCH 424/480] Fix markup to not add parens to the "hashlib.md5" constructor when the object is meant, not the call. --- Doc/library/hmac.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index e962ff0de71..f88a03e0f97 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -20,7 +20,7 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. Return a new hmac object. If *msg* is present, the method call ``update(msg)`` is made. *digestmod* is the digest constructor or module for the HMAC object to - use. It defaults to the :func:`hashlib.md5` constructor. + use. It defaults to the :data:`hashlib.md5` constructor. An HMAC object has the following methods: From 0ffae6b5bca637f631d6b3091d21ac179768316d Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 09:50:36 +0200 Subject: [PATCH 425/480] Fix prefix_chars not being applied in help text example (reported by John Kooker on docs@) --- Doc/library/argparse.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 61cb499b132..0dc2ab444d9 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -579,10 +579,10 @@ the help options:: >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='+/') >>> parser.print_help() - usage: PROG [-h] + usage: PROG [+h] optional arguments: - -h, --help show this help message and exit + +h, ++help show this help message and exit The add_argument() method From f768c985d3f1175c5322a638099bc7332af12617 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 09:52:55 +0200 Subject: [PATCH 426/480] Clarify docs for os.path.getctime on Unix: it is the inode (metadata) change time --- Doc/library/os.path.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index c84aaf29e10..e9b98132e8a 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -147,7 +147,7 @@ the :mod:`glob` module.) .. function:: getctime(path) Return the system's ctime which, on some systems (like Unix) is the time of the - last change, and, on others (like Windows), is the creation time for *path*. + last metadata change, and, on others (like Windows), is the creation time for *path*. The return value is a number giving the number of seconds since the epoch (see the :mod:`time` module). Raise :exc:`os.error` if the file does not exist or is inaccessible. From 1f0b03365977a507bb6cc40d943f03f3f5797809 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 10:02:07 +0200 Subject: [PATCH 427/480] Fix some PEP8 in curses HOWTO. --- Doc/howto/curses.rst | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 1fc10c763da..74c1f2a4402 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -144,8 +144,8 @@ window, but you might wish to divide the screen into smaller windows, in order to redraw or clear them separately. The :func:`newwin` function creates a new window of a given size, returning the new window object. :: - begin_x = 20 ; begin_y = 7 - height = 5 ; width = 40 + begin_x = 20; begin_y = 7 + height = 5; width = 40 win = curses.newwin(height, width, begin_y, begin_x) A word about the coordinate system used in curses: coordinates are always passed @@ -184,11 +184,13 @@ displayed. :: # explained in the next section for y in range(0, 100): for x in range(0, 100): - try: pad.addch(y,x, ord('a') + (x*x+y*y) % 26 ) - except curses.error: pass + try: + pad.addch(y,x, ord('a') + (x*x+y*y) % 26) + except curses.error: + pass # Displays a section of the pad in the middle of the screen - pad.refresh( 0,0, 5,5, 20,75) + pad.refresh(0,0, 5,5, 20,75) The :func:`refresh` call displays a section of the pad in the rectangle extending from coordinate (5,5) to coordinate (20,75) on the screen; the upper @@ -321,7 +323,7 @@ again, such combinations are not guaranteed to work on all terminals. An example, which displays a line of text using color pair 1:: - stdscr.addstr( "Pretty text", curses.color_pair(1) ) + stdscr.addstr("Pretty text", curses.color_pair(1)) stdscr.refresh() As I said before, a color pair consists of a foreground and background color. @@ -343,7 +345,7 @@ When you change a color pair, any text already displayed using that color pair will change to the new colors. You can also display new text in this color with:: - stdscr.addstr(0,0, "RED ALERT!", curses.color_pair(1) ) + stdscr.addstr(0,0, "RED ALERT!", curses.color_pair(1)) Very fancy terminals can change the definitions of the actual colors to a given RGB value. This lets you change color 1, which is usually red, to purple or @@ -381,9 +383,12 @@ your program will look something like this:: while 1: c = stdscr.getch() - if c == ord('p'): PrintDocument() - elif c == ord('q'): break # Exit the while() - elif c == curses.KEY_HOME: x = y = 0 + if c == ord('p'): + PrintDocument() + elif c == ord('q'): + break # Exit the while() + elif c == curses.KEY_HOME: + x = y = 0 The :mod:`curses.ascii` module supplies ASCII class membership functions that take either integer or 1-character-string arguments; these may be useful in @@ -433,4 +438,3 @@ If you write an interesting little program, feel free to contribute it as another demo. We can always use more of them! The ncurses FAQ: http://invisible-island.net/ncurses/ncurses.faq.html - From 39bbdc33d292f98cf22b02c19ab76304e2ae5ebd Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 10:11:12 +0200 Subject: [PATCH 428/480] Fix small copy-paste typo in ossaudiodev setparameters() example (found by Ken Housley on docs@). --- Doc/library/ossaudiodev.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst index c5404deb9c7..1a65d246a8c 100644 --- a/Doc/library/ossaudiodev.rst +++ b/Doc/library/ossaudiodev.rst @@ -296,7 +296,7 @@ simple calculations. fmt = dsp.setfmt(fmt) channels = dsp.channels(channels) - rate = dsp.rate(channels) + rate = dsp.rate(rate) .. method:: oss_audio_device.bufsize() From 84dc8dc0b0405135fe1ebf1d8aef318498378415 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 10:22:45 +0200 Subject: [PATCH 429/480] Closes #18646: improve lambda docs in tutorial. Original patch by Terry Reedy. --- Doc/tutorial/controlflow.rst | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index aad26720119..82fbb82053c 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -541,17 +541,16 @@ In the same fashion, dictionaries can deliver keyword arguments with the ``**``\ .. _tut-lambda: -Lambda Forms ------------- - -By popular demand, a few features commonly found in functional programming -languages like Lisp have been added to Python. With the :keyword:`lambda` -keyword, small anonymous functions can be created. Here's a function that -returns the sum of its two arguments: ``lambda a, b: a+b``. Lambda forms can be -used wherever function objects are required. They are syntactically restricted -to a single expression. Semantically, they are just syntactic sugar for a -normal function definition. Like nested function definitions, lambda forms can -reference variables from the containing scope:: +Lambda Expressions +------------------ + +Small anonymous functions can be created with the :keyword:`lambda` keyword. +This function returns the sum of its two arguments: ``lambda a, b: a+b``. +Lambda forms can be used wherever function objects are required. They are +syntactically restricted to a single expression. Semantically, they are just +syntactic sugar for a normal function definition. Like nested function +definitions, lambda functions can reference variables from the containing +scope:: >>> def make_incrementor(n): ... return lambda x: x + n @@ -562,6 +561,14 @@ reference variables from the containing scope:: >>> f(1) 43 +The above example uses a lambda expression to return a function. Another use +is to pass a small function as an argument:: + + >>> pairs = [(1, 'one'), (2, 'two'), (3, 'three'), (4, 'four')] + >>> pairs.sort(key=lambda pair: pair[1]) + >>> pairs + [(4, 'four'), (1, 'one'), (3, 'three'), (2, 'two')] + .. _tut-docstrings: From f814aad47143ebe4bd7c53ae516bab3127a2e505 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 10:26:58 +0200 Subject: [PATCH 430/480] Use "lambda expression" as preferred to "lambda form". --- Doc/faq/design.rst | 10 +++++----- Doc/reference/compound_stmts.rst | 8 ++++---- Doc/reference/expressions.rst | 13 ++++++------- Doc/tutorial/controlflow.rst | 2 +- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 73c90af0372..40babecdb70 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -372,20 +372,20 @@ Answer 2: Fortunately, there is `Stackless Python `_, which has a completely redesigned interpreter loop that avoids the C stack. -Why can't lambda forms contain statements? ------------------------------------------- +Why can't lambda expressions contain statements? +------------------------------------------------ -Python lambda forms cannot contain statements because Python's syntactic +Python lambda expressions cannot contain statements because Python's syntactic framework can't handle statements nested inside expressions. However, in Python, this is not a serious problem. Unlike lambda forms in other languages, where they add functionality, Python lambdas are only a shorthand notation if you're too lazy to define a function. Functions are already first class objects in Python, and can be declared in a -local scope. Therefore the only advantage of using a lambda form instead of a +local scope. Therefore the only advantage of using a lambda instead of a locally-defined function is that you don't need to invent a name for the function -- but that's just a local variable to which the function object (which -is exactly the same type of object that a lambda form yields) is assigned! +is exactly the same type of object that a lambda expression yields) is assigned! Can Python be compiled to machine code, C or some other language? diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 7b6c3e22246..4a9ad08200a 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -509,14 +509,14 @@ receiving any excess positional parameters, defaulting to the empty tuple. If the form "``**identifier``" is present, it is initialized to a new dictionary receiving any excess keyword arguments, defaulting to a new empty dictionary. -.. index:: pair: lambda; form +.. index:: pair: lambda; expression It is also possible to create anonymous functions (functions not bound to a -name), for immediate use in expressions. This uses lambda forms, described in -section :ref:`lambda`. Note that the lambda form is merely a shorthand for a +name), for immediate use in expressions. This uses lambda expressions, described in +section :ref:`lambda`. Note that the lambda expression is merely a shorthand for a simplified function definition; a function defined in a ":keyword:`def`" statement can be passed around or assigned to another name just like a function -defined by a lambda form. The ":keyword:`def`" form is actually more powerful +defined by a lambda expression. The ":keyword:`def`" form is actually more powerful since it allows the execution of multiple statements. **Programmer's note:** Functions are first-class objects. A "``def``" form diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 4b05c374f18..1e46af2ad32 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -184,7 +184,7 @@ brackets: list_comprehension: `expression` `list_for` list_for: "for" `target_list` "in" `old_expression_list` [`list_iter`] old_expression_list: `old_expression` [("," `old_expression`)+ [","]] - old_expression: `or_test` | `old_lambda_form` + old_expression: `or_test` | `old_lambda_expr` list_iter: `list_for` | `list_if` list_if: "if" `old_expression` [`list_iter`] @@ -1255,7 +1255,7 @@ Conditional Expressions .. productionlist:: conditional_expression: `or_test` ["if" `or_test` "else" `expression`] - expression: `conditional_expression` | `lambda_form` + expression: `conditional_expression` | `lambda_expr` Conditional expressions (sometimes called a "ternary operator") have the lowest priority of all Python operations. @@ -1275,14 +1275,13 @@ Lambdas .. index:: pair: lambda; expression - pair: lambda; form pair: anonymous; function .. productionlist:: - lambda_form: "lambda" [`parameter_list`]: `expression` - old_lambda_form: "lambda" [`parameter_list`]: `old_expression` + lambda_expr: "lambda" [`parameter_list`]: `expression` + old_lambda_expr: "lambda" [`parameter_list`]: `old_expression` -Lambda forms (lambda expressions) have the same syntactic position as +Lambda expressions (sometimes called lambda forms) have the same syntactic position as expressions. They are a shorthand to create anonymous functions; the expression ``lambda arguments: expression`` yields a function object. The unnamed object behaves like a function object defined with :: @@ -1291,7 +1290,7 @@ behaves like a function object defined with :: return expression See section :ref:`function` for the syntax of parameter lists. Note that -functions created with lambda forms cannot contain statements. +functions created with lambda expressions cannot contain statements. .. _exprlists: diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 82fbb82053c..4de9ec927f3 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -546,7 +546,7 @@ Lambda Expressions Small anonymous functions can be created with the :keyword:`lambda` keyword. This function returns the sum of its two arguments: ``lambda a, b: a+b``. -Lambda forms can be used wherever function objects are required. They are +Lambda functions can be used wherever function objects are required. They are syntactically restricted to a single expression. Semantically, they are just syntactic sugar for a normal function definition. Like nested function definitions, lambda functions can reference variables from the containing From 6fb3542594288af184cf24b7a8400a617f8104f1 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 11:02:38 +0200 Subject: [PATCH 431/480] Small clarification in extending docs. --- Doc/extending/extending.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index eb18a46f871..792586d9364 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -510,7 +510,7 @@ or more format codes between parentheses. For example:: value of the Python function. :c:func:`PyObject_CallObject` is "reference-count-neutral" with respect to its arguments. In the example a new tuple was created to serve as the argument list, which is :c:func:`Py_DECREF`\ --ed immediately after the call. +-ed immediately after the :c:func:`PyObject_CallObject` call. The return value of :c:func:`PyObject_CallObject` is "new": either it is a brand new object, or it is an existing object whose reference count has been From cc0f9dec1ea8f036af4977e9971f04a298ed9611 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 11:08:24 +0200 Subject: [PATCH 432/480] Fix wrong Python highlighting in directory hierarchy code block. --- Doc/tutorial/modules.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index bcbb761b785..1e792d1b8f5 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -368,7 +368,9 @@ There are also many different operations you might want to perform on sound data (such as mixing, adding echo, applying an equalizer function, creating an artificial stereo effect), so in addition you will be writing a never-ending stream of modules to perform these operations. Here's a possible structure for -your package (expressed in terms of a hierarchical filesystem):: +your package (expressed in terms of a hierarchical filesystem): + +.. code-block:: text sound/ Top-level package __init__.py Initialize the sound package From 1255416822d53ab2096b89976ae68bdd4974091e Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 11:08:57 +0200 Subject: [PATCH 433/480] Fix merging mistake: wrong Python version. --- Doc/tutorial/interpreter.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 6bdc1c5ba4e..25380e48a7b 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -252,7 +252,7 @@ of your user site-packages directory. Start Python and run this code: >>> import site >>> site.getusersitepackages() - '/home/user/.local/lib/python3.2/site-packages' + '/home/user/.local/lib/python2.7/site-packages' Now you can create a file named :file:`usercustomize.py` in that directory and put anything you want in it. It will affect every invocation of Python, unless From c97f8e6a819bf658c30d0455682ca8f6ed1e3abf Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 11:10:49 +0200 Subject: [PATCH 434/480] Fix deprecated link to Python Testing Tools Taxonomy. Reported by Daniel Greenfeld on docs@. --- Doc/library/unittest.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index e5faa9a50f2..999b024e696 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -91,7 +91,7 @@ need to derive from a specific class. Third-party unittest frameworks with a lighter-weight syntax for writing tests. For example, ``assert func(10) == 42``. - `The Python Testing Tools Taxonomy `_ + `The Python Testing Tools Taxonomy `_ An extensive list of Python testing tools including functional testing frameworks and mock object libraries. From bea7fc9356a050c7f6086b2a235afcaf0fa877a4 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 11:12:29 +0200 Subject: [PATCH 435/480] Fix missing period. --- Doc/distutils/apiref.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index f34a8eaa673..480feabfba6 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -729,7 +729,7 @@ This module provides the following functions. .. method:: CCompiler.execute(func, args[, msg=None, level=1]) - Invokes :func:`distutils.util.execute` This method invokes a Python function + Invokes :func:`distutils.util.execute`. This method invokes a Python function *func* with the given arguments *args*, after logging and taking into account the *dry_run* flag. From 1080852b135dc5ac473e16ba99d6d4390d0a4b18 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 11:24:48 +0200 Subject: [PATCH 436/480] Clarify section about source code encoding. --- Doc/tutorial/interpreter.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 25380e48a7b..5140b433848 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -185,8 +185,8 @@ encodings can be found in the Python Library Reference, in the section on For example, to write Unicode literals including the Euro currency symbol, the ISO-8859-15 encoding can be used, with the Euro symbol having the ordinal value -164. This script will print the value 8364 (the Unicode codepoint corresponding -to the Euro symbol) and then exit:: +164. This script, when saved in the ISO-8859-15 encoding, will print the value +8364 (the Unicode codepoint corresponding to the Euro symbol) and then exit:: # -*- coding: iso-8859-15 -*- From 17ba61da6cc02db4a2a83b1adf0e5b9ab60ea542 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 11:41:36 +0200 Subject: [PATCH 437/480] Small logical fix in filter() example description. --- Doc/tutorial/datastructures.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index e6786cc0e27..28e6ad71b95 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -171,7 +171,7 @@ There are three built-in functions that are very useful when used with lists: the sequence for which ``function(item)`` is true. If *sequence* is a :class:`string` or :class:`tuple`, the result will be of the same type; otherwise, it is always a :class:`list`. For example, to compute a sequence of -numbers not divisible by 2 and 3:: +numbers not divisible by 2 or 3:: >>> def f(x): return x % 2 != 0 and x % 3 != 0 ... From 928287ef1558c47825276258541435a2cc132864 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 11:57:13 +0200 Subject: [PATCH 438/480] Fix small grammar mistake. --- Doc/glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 219a6caf2ce..3af49fd0297 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -664,7 +664,7 @@ Glossary statement A statement is part of a suite (a "block" of code). A statement is either - an :term:`expression` or a one of several constructs with a keyword, such + an :term:`expression` or one of several constructs with a keyword, such as :keyword:`if`, :keyword:`while` or :keyword:`for`. struct sequence From bef939a9eeb790b9f0b06f643a6747f27c2efea6 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 12:08:14 +0200 Subject: [PATCH 439/480] Closes #15956: improve documentation of named groups and how to reference them. --- Doc/library/re.rst | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 1af4cf2011d..53d18903f1a 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -237,21 +237,32 @@ The special characters are: ``(?P...)`` Similar to regular parentheses, but the substring matched by the group is - accessible within the rest of the regular expression via the symbolic group - name *name*. Group names must be valid Python identifiers, and each group - name must be defined only once within a regular expression. A symbolic group - is also a numbered group, just as if the group were not named. So the group - named ``id`` in the example below can also be referenced as the numbered group - ``1``. - - For example, if the pattern is ``(?P[a-zA-Z_]\w*)``, the group can be - referenced by its name in arguments to methods of match objects, such as - ``m.group('id')`` or ``m.end('id')``, and also by name in the regular - expression itself (using ``(?P=id)``) and replacement text given to - ``.sub()`` (using ``\g``). + accessible via the symbolic group name *name*. Group names must be valid + Python identifiers, and each group name must be defined only once within a + regular expression. A symbolic group is also a numbered group, just as if + the group were not named. + + Named groups can be referenced in three contexts. If the pattern is + ``(?P['"]).*?(?P=quote)`` (i.e. matching a string quoted with either + single or double quotes): + + +---------------------------------------+----------------------------------+ + | Context of reference to group "quote" | Ways to reference it | + +=======================================+==================================+ + | in the same pattern itself | * ``(?P=quote)`` (as shown) | + | | * ``\1`` | + +---------------------------------------+----------------------------------+ + | when processing match object ``m`` | * ``m.group('quote')`` | + | | * ``m.end('quote')`` (etc.) | + +---------------------------------------+----------------------------------+ + | in a string passed to the ``repl`` | * ``\g`` | + | argument of ``re.sub()`` | * ``\g<1>`` | + | | * ``\1`` | + +---------------------------------------+----------------------------------+ ``(?P=name)`` - Matches whatever text was matched by the earlier group named *name*. + A backreference to a named group; it matches whatever text was matched by the + earlier group named *name*. ``(?#...)`` A comment; the contents of the parentheses are simply ignored. @@ -331,7 +342,8 @@ the second character. For example, ``\$`` matches the character ``'$'``. depends on the values of the ``UNICODE`` and ``LOCALE`` flags. For example, ``r'\bfoo\b'`` matches ``'foo'``, ``'foo.'``, ``'(foo)'``, ``'bar foo baz'`` but not ``'foobar'`` or ``'foo3'``. - Inside a character range, ``\b`` represents the backspace character, for compatibility with Python's string literals. + Inside a character range, ``\b`` represents the backspace character, for + compatibility with Python's string literals. ``\B`` Matches the empty string, but only when it is *not* at the beginning or end of a @@ -642,7 +654,8 @@ form. when not adjacent to a previous match, so ``sub('x*', '-', 'abc')`` returns ``'-a-b-c-'``. - In addition to character escapes and backreferences as described above, + In string-type *repl* arguments, in addition to the character escapes and + backreferences described above, ``\g`` will use the substring matched by the group named ``name``, as defined by the ``(?P...)`` syntax. ``\g`` uses the corresponding group number; ``\g<2>`` is therefore equivalent to ``\2``, but isn't ambiguous From 48fe3baab7f452596739157941c3f0d97805a63a Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 12:31:06 +0200 Subject: [PATCH 440/480] Fix strange deprecation version. --- Doc/library/pipes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/pipes.rst b/Doc/library/pipes.rst index 6d7dff8c378..415d5c766b9 100644 --- a/Doc/library/pipes.rst +++ b/Doc/library/pipes.rst @@ -35,7 +35,7 @@ Example:: .. function:: quote(s) - .. deprecated:: 1.6 + .. deprecated:: 2.7 Prior to Python 2.7, this function was not publicly documented. It is finally exposed publicly in Python 3.3 as the :func:`quote ` function in the :mod:`shlex` module. From 994b68935d71635784bc9278a12543de783b3af3 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 12:33:20 +0200 Subject: [PATCH 441/480] Fix typo. --- Doc/library/gzip.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 4dd479ecf06..aa89c2cef34 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -38,7 +38,7 @@ The module defines the following items: object. When *fileobj* is not ``None``, the *filename* argument is only used to be - included in the :program:`gzip` file header, which may includes the original + included in the :program:`gzip` file header, which may include the original filename of the uncompressed file. It defaults to the filename of *fileobj*, if discernible; otherwise, it defaults to the empty string, and in this case the original filename is not included in the header. From cc8cac725d3bcb27db2cadbfe281f95809137375 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 12:36:39 +0200 Subject: [PATCH 442/480] Counter: fix recipe for "n least common elements". Found by Mikhail Golubev on docs@. --- Doc/library/collections.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index b7628bb8deb..838c507caec 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -146,7 +146,7 @@ Common patterns for working with :class:`Counter` objects:: dict(c) # convert to a regular dictionary c.items() # convert to a list of (elem, cnt) pairs Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs - c.most_common()[:-n:-1] # n least common elements + c.most_common()[:-n-1:-1] # n least common elements c += Counter() # remove zero and negative counts Several mathematical operations are provided for combining :class:`Counter` From 1706cfd90816fd56dae31e5826dc41e41efc51bf Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 12:38:44 +0200 Subject: [PATCH 443/480] Fix SQLite datatype name: it is REAL, not FLOAT. Found by Richard Kelsall on docs@. --- Doc/library/sqlite3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index c32bb907114..6ca50d8a553 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -179,7 +179,7 @@ Module functions and constants For the *isolation_level* parameter, please see the :attr:`Connection.isolation_level` property of :class:`Connection` objects. - SQLite natively supports only the types TEXT, INTEGER, FLOAT, BLOB and NULL. If + SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL. If you want to use other types you must add support for them yourself. The *detect_types* parameter and the using custom **converters** registered with the module-level :func:`register_converter` function allow you to easily do that. From 80d985d014c07afed693a695acad7df37c9e5c13 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 12:42:18 +0200 Subject: [PATCH 444/480] Fix minor bug in httplib example. Found by Alex MacAulay on docs@. --- Doc/library/httplib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/httplib.rst b/Doc/library/httplib.rst index 472fa61e13d..fcdfbc015d4 100644 --- a/Doc/library/httplib.rst +++ b/Doc/library/httplib.rst @@ -626,6 +626,6 @@ be created via ``PUT`` request. Here is an example session that shows how to do >>> conn = httplib.HTTPConnection("localhost", 8080) >>> conn.request("PUT", "/file", BODY) >>> response = conn.getresponse() - >>> print resp.status, response.reason + >>> print response.status, response.reason 200, OK From 82fe76cd48abe910e7793df4618dbeb586537efc Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 12:46:13 +0200 Subject: [PATCH 445/480] PythonCAD is now on PyQt, use Wing as a prominent PyGtk example. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by Helge Stenström on docs@. --- Doc/library/othergui.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst index 69df9df85eb..28b57717648 100644 --- a/Doc/library/othergui.rst +++ b/Doc/library/othergui.rst @@ -14,7 +14,7 @@ available for Python: the C one. It comes with many more widgets than Tkinter provides, and has good Python-specific reference documentation. There are also bindings to `GNOME `_. One well known PyGTK application is - `PythonCAD `_. An online `tutorial + `WingIDE `_. An online `tutorial `_ is available. `PyQt `_ From 13e19c1826afce6d6fa65f8d14cb93b819d3d77f Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 12:58:20 +0200 Subject: [PATCH 446/480] Fix example in backreference description. Found by Alexander Heger on docs@. --- Doc/library/re.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 53d18903f1a..91afbd497c9 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -322,7 +322,7 @@ the second character. For example, ``\$`` matches the character ``'$'``. ``\number`` Matches the contents of the group of the same number. Groups are numbered starting from 1. For example, ``(.+) \1`` matches ``'the the'`` or ``'55 55'``, - but not ``'the end'`` (note the space after the group). This special sequence + but not ``'thethe'`` (note the space after the group). This special sequence can only be used to match one of the first 99 groups. If the first digit of *number* is 0, or *number* is 3 octal digits long, it will not be interpreted as a group match, but as the character with octal value *number*. Inside the From 9677e749237737056db43a356d2955281f818453 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 13:09:59 +0200 Subject: [PATCH 447/480] Fix missing class name in markup. Found by Tanky Woo on docs@. --- Doc/library/argparse.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 0dc2ab444d9..a1d58c0b8e7 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1666,7 +1666,7 @@ Argument groups Mutual exclusion ^^^^^^^^^^^^^^^^ -.. method:: add_mutually_exclusive_group(required=False) +.. method:: ArgumentParser.add_mutually_exclusive_group(required=False) Create a mutually exclusive group. :mod:`argparse` will make sure that only one of the arguments in the mutually exclusive group was present on the From 08ff0bec9617024848a47f78cd30e4df3c8d2012 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 13:14:10 +0200 Subject: [PATCH 448/480] Minor improvement: add indication that the main function goes on. --- Doc/extending/extending.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 792586d9364..97c6d03e174 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -372,6 +372,8 @@ to :c:func:`Py_Initialize`:: /* Add a static module */ initspam(); + ... + An example may be found in the file :file:`Demo/embed/demo.c` in the Python source distribution. From 7e5419652de5cfa5a0ebd489a20ad344d4bce49a Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 13:17:04 +0200 Subject: [PATCH 449/480] Fix order of required StreamReader/StreamWriter base classes. Spotted by Edward Welbourne on docs@. --- Doc/library/codecs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 61da3ed9e7e..d283edeb4cf 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -66,7 +66,7 @@ It defines the following functions: ``factory(stream, errors='strict')`` The factory functions must return objects providing the interfaces defined by - the base classes :class:`StreamWriter` and :class:`StreamReader`, respectively. + the base classes :class:`StreamReader` and :class:`StreamWriter`, respectively. Stream codecs can maintain state. Possible values for errors are From 23e6b957ccba44f6a874e00c51d5a0804214b9ff Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 13:20:49 +0200 Subject: [PATCH 450/480] Fix link to the "pexports" tool. Found by Joe Aikkaravelil on docs@. --- Doc/install/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/install/index.rst b/Doc/install/index.rst index 8b51b97e11a..159f84816da 100644 --- a/Doc/install/index.rst +++ b/Doc/install/index.rst @@ -1042,7 +1042,7 @@ These compilers require some special libraries. This task is more complex than for Borland's C++, because there is no program to convert the library. First you have to create a list of symbols which the Python DLL exports. (You can find a good program for this task at -http://www.emmestech.com/software/pexports-0.43/download_pexports.html). +http://sourceforge.net/projects/mingw/files/MinGW/Extension/pexports/). .. I don't understand what the next line means. --amk .. (inclusive the references on data structures.) From 8e5b7fee176b93b4675234df3bcb612eb57d88b8 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 18:11:32 +0200 Subject: [PATCH 451/480] Closes #12350: clarify blocks/block size members of stat result. --- Doc/library/os.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index d4912ce9c34..64d7874ef0a 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1386,8 +1386,8 @@ Files and Directories On some Unix systems (such as Linux), the following attributes may also be available: - * :attr:`st_blocks` - number of blocks allocated for file - * :attr:`st_blksize` - filesystem blocksize + * :attr:`st_blocks` - number of 512-byte blocks allocated for file + * :attr:`st_blksize` - filesystem blocksize for efficient file system I/O * :attr:`st_rdev` - type of device if an inode device * :attr:`st_flags` - user defined flags for file From 986ceff512059b0f70809a52056805632beda593 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 18:17:56 +0200 Subject: [PATCH 452/480] Closes #19181: replace non-existing host ftp.cwi.nl with ftp.debian.org in ftplib example. --- Doc/library/ftplib.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 988278952b5..559e8a21ddc 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -23,16 +23,17 @@ see Internet :rfc:`959`. Here's a sample session using the :mod:`ftplib` module:: >>> from ftplib import FTP - >>> ftp = FTP('ftp.cwi.nl') # connect to host, default port - >>> ftp.login() # user anonymous, passwd anonymous@ - >>> ftp.retrlines('LIST') # list directory contents - total 24418 - drwxrwsr-x 5 ftp-usr pdmaint 1536 Mar 20 09:48 . - dr-xr-srwt 105 ftp-usr pdmaint 1536 Mar 21 14:32 .. - -rw-r--r-- 1 ftp-usr pdmaint 5305 Mar 20 09:48 INDEX - . - . - . + >>> ftp = FTP('ftp.debian.org') # connect to host, default port + >>> ftp.login() # user anonymous, passwd anonymous@ + '230 Login successful.' + >>> ftp.cwd('debian') # change into "debian" directory + >>> ftp.retrlines('LIST') # list directory contents + -rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18 README + ... + drwxr-sr-x 5 1176 1176 4096 Dec 19 2000 pool + drwxr-sr-x 4 1176 1176 4096 Nov 17 2008 project + drwxr-xr-x 3 1176 1176 4096 Oct 10 2012 tools + '226 Directory send OK.' >>> ftp.retrbinary('RETR README', open('README', 'wb').write) '226 Transfer complete.' >>> ftp.quit() @@ -387,7 +388,8 @@ FTP_TLS Objects .. method:: FTP_TLS.auth() - Set up secure control connection by using TLS or SSL, depending on what specified in :meth:`ssl_version` attribute. + Set up secure control connection by using TLS or SSL, depending on what + specified in :meth:`ssl_version` attribute. .. method:: FTP_TLS.prot_p() @@ -396,5 +398,3 @@ FTP_TLS Objects .. method:: FTP_TLS.prot_c() Set up clear text data connection. - - From 34acdeaa519e1dab32b87234303adfeb153edb09 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 18:20:31 +0200 Subject: [PATCH 453/480] Closes #19177: replace dead link to SSL/TLS introduction with the version from Apache. --- Doc/library/ssl.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 01072c458bb..550c249894a 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -641,10 +641,10 @@ And go back to listening for new client connections. .. seealso:: Class :class:`socket.socket` - Documentation of underlying :mod:`socket` class + Documentation of underlying :mod:`socket` class - `TLS (Transport Layer Security) and SSL (Secure Socket Layer) `_ - Debby Koren + `SSL/TLS Strong Encryption: An Introduction `_ + Intro from the Apache webserver documentation `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management `_ Steve Kent From 4a313be9cab2984a3d9ee2580106f10b05b76af4 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 18:26:36 +0200 Subject: [PATCH 454/480] The "FAQ" link at the end was changed to a Wikipedia link; reflect that. --- Doc/library/hashlib.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 063ad59716b..aa8e6e81ff5 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -25,12 +25,14 @@ digest are interchangeable. Older algorithms were called message digests. The modern term is secure hash. .. note:: - If you want the adler32 or crc32 hash functions they are available in + + If you want the adler32 or crc32 hash functions, they are available in the :mod:`zlib` module. .. warning:: - Some algorithms have known hash collision weaknesses, see the FAQ at the end. + Some algorithms have known hash collision weaknesses, refer to the "See + also" section at the end. There is one constructor method named for each type of :dfn:`hash`. All return a hash object with the same simple interface. For example: use :func:`sha1` to From f8dd70f912a1b4aabfdccae302df80976391694f Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 18:36:34 +0200 Subject: [PATCH 455/480] Closes #17725: small grammar fix. --- Doc/extending/index.rst | 12 ++++++------ Misc/ACKS | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst index 75cf4c5d316..61129439958 100644 --- a/Doc/extending/index.rst +++ b/Doc/extending/index.rst @@ -5,12 +5,12 @@ ################################################## This document describes how to write modules in C or C++ to extend the Python -interpreter with new modules. Those modules can define new functions but also -new object types and their methods. The document also describes how to embed -the Python interpreter in another application, for use as an extension language. -Finally, it shows how to compile and link extension modules so that they can be -loaded dynamically (at run time) into the interpreter, if the underlying -operating system supports this feature. +interpreter with new modules. Those modules can not only define new functions +but also new object types and their methods. The document also describes how +to embed the Python interpreter in another application, for use as an extension +language. Finally, it shows how to compile and link extension modules so that +they can be loaded dynamically (at run time) into the interpreter, if the +underlying operating system supports this feature. This document assumes basic knowledge about Python. For an informal introduction to the language, see :ref:`tutorial-index`. :ref:`reference-index` diff --git a/Misc/ACKS b/Misc/ACKS index cbef1f00fcf..769624c6cf3 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -959,6 +959,7 @@ Ionel Simionescu Kirill Simonov Nathan Paul Simons Guilherme Simões +Kyle Simpson Ravi Sinha Janne Sinkkonen Ng Pheng Siong From 42dc3f1c862273c3b82f7870c0f1d31429ffad9f Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 18:43:19 +0200 Subject: [PATCH 456/480] Closes #15213: update comment for _PyOS_URandom --- Doc/library/os.rst | 5 +++-- Python/random.c | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 64d7874ef0a..3d48722d1cf 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2501,8 +2501,9 @@ Miscellaneous Functions This function returns random bytes from an OS-specific randomness source. The returned data should be unpredictable enough for cryptographic applications, though its exact quality depends on the OS implementation. On a UNIX-like - system this will query /dev/urandom, and on Windows it will use CryptGenRandom. - If a randomness source is not found, :exc:`NotImplementedError` will be raised. + system this will query ``/dev/urandom``, and on Windows it will use + ``CryptGenRandom()``. If a randomness source is not found, + :exc:`NotImplementedError` will be raised. For an easy-to-use interface to the random number generator provided by your platform, please see :class:`random.SystemRandom`. diff --git a/Python/random.c b/Python/random.c index 73e3cc3551f..d615923dd2f 100644 --- a/Python/random.c +++ b/Python/random.c @@ -224,8 +224,9 @@ lcg_urandom(unsigned int x0, unsigned char *buffer, size_t size) } } -/* Fill buffer with size pseudo-random bytes, not suitable for cryptographic - use, from the operating random number generator (RNG). +/* Fill buffer with size pseudo-random bytes from the operating system random + number generator (RNG). It is suitable for for most cryptographic purposes + except long living private keys for asymmetric encryption. Return 0 on success, raise an exception and return -1 on error. */ int From 90549b9ea7d01f6bf06dcb2e5a91fd8fac381b6d Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 18:51:39 +0200 Subject: [PATCH 457/480] Closes #11807: document argparse add_subparsers method better. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch by Filip GruszczyÅ„ski. --- Doc/library/argparse.rst | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index a1d58c0b8e7..28ecee9e105 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1432,7 +1432,10 @@ Other utilities Sub-commands ^^^^^^^^^^^^ -.. method:: ArgumentParser.add_subparsers() +.. method:: ArgumentParser.add_subparsers([title], [description], [prog], \ + [parser_class], [action], \ + [option_string], [dest], [help], \ + [metavar]) Many programs split up their functionality into a number of sub-commands, for example, the ``svn`` program can invoke sub-commands like ``svn @@ -1446,6 +1449,30 @@ Sub-commands command name and any :class:`ArgumentParser` constructor arguments, and returns an :class:`ArgumentParser` object that can be modified as usual. + Description of parameters: + + * title - title for the sub-parser group in help output; by default + "subcommands" if description is provided, otherwise uses title for + positional arguments + + * description - description for the sub-parser group in help output, by + default None + + * prog - usage information that will be displayed with sub-command help, + by default the name of the program and any positional arguments before the + subparser argument + + * parser_class - class which will be used to create sub-parser instances, by + default the class of the current parser (e.g. ArgumentParser) + + * dest - name of the attribute under which sub-command name will be + stored; by default None and no value is stored + + * help - help for sub-parser group in help output, by default None + + * metavar - string presenting available sub-commands in help; by default it + is None and presents sub-commands in form {cmd1, cmd2, ..} + Some example usage:: >>> # create the top-level parser From 820b8cec9752fa58aad1624721e285c62ac0dcde Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 19:01:21 +0200 Subject: [PATCH 458/480] Closes #15432: GzipFile mtime argument was added in 2.7. --- Doc/library/gzip.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index aa89c2cef34..e26fe286c40 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -77,6 +77,9 @@ The module defines the following items: .. versionchanged:: 2.7 Support for zero-padded files was added. + .. versionadded:: 2.7 + The *mtime* argument. + .. function:: open(filename[, mode[, compresslevel]]) From a132d20e1c100dc638f00301aab40c9a459ea025 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 19:14:35 +0200 Subject: [PATCH 459/480] Fix typo in function name. --- Doc/library/operator.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 9f17e987675..244da144f21 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -514,7 +514,7 @@ expect a function argument. return resolve_attr(obj, attr) else: def g(obj): - return tuple(resolve_att(obj, attr) for attr in items) + return tuple(resolve_attr(obj, attr) for attr in items) return g def resolve_attr(obj, attr): From bc6f850045a5a4223329cfded92231aa168c7209 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 19:19:18 +0200 Subject: [PATCH 460/480] Fix: win32_ver returns a 4-tuple. Found by Andrew Berg on docs@. --- Doc/library/platform.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 26f587eaf3c..cb7144a64ad 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -197,8 +197,8 @@ Windows Platform .. function:: win32_ver(release='', version='', csd='', ptype='') Get additional version information from the Windows Registry and return a tuple - ``(version, csd, ptype)`` referring to version number, CSD level - (service pack) and OS type (multi/single processor). + ``(release, version, csd, ptype)`` referring to OS release, version number, + CSD level (service pack) and OS type (multi/single processor). As a hint: *ptype* is ``'Uniprocessor Free'`` on single processor NT machines and ``'Multiprocessor Free'`` on multi processor machines. The *'Free'* refers From b98db0e27445bfd79086f260f619d82d8dd8c15a Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 19:21:14 +0200 Subject: [PATCH 461/480] Fix typo in example file name. Found by John Bokma on docs@. --- Doc/tutorial/modules.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 1e792d1b8f5..f5b811452ca 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -467,7 +467,7 @@ list of module names that should be imported when ``from package import *`` is encountered. It is up to the package author to keep this list up-to-date when a new version of the package is released. Package authors may also decide not to support it, if they don't see a use for importing \* from their package. For -example, the file :file:`sounds/effects/__init__.py` could contain the following +example, the file :file:`sound/effects/__init__.py` could contain the following code:: __all__ = ["echo", "surround", "reverse"] From b92117b42bc374b0b72081f6118d256ff428ece1 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 6 Oct 2013 19:33:56 +0200 Subject: [PATCH 462/480] Use the platform-independent way of calling a script from the shell: "python prog.py", not "prog.py" Found by Micheal Wells on docs@. --- Doc/library/argparse.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 28ecee9e105..489df839789 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -46,7 +46,7 @@ produces either the sum or the max:: Assuming the Python code above is saved into a file called ``prog.py``, it can be run at the command line and provides useful help messages:: - $ prog.py -h + $ python prog.py -h usage: prog.py [-h] [--sum] N [N ...] Process some integers. @@ -61,15 +61,15 @@ be run at the command line and provides useful help messages:: When run with the appropriate arguments, it prints either the sum or the max of the command-line integers:: - $ prog.py 1 2 3 4 + $ python prog.py 1 2 3 4 4 - $ prog.py 1 2 3 4 --sum + $ python prog.py 1 2 3 4 --sum 10 If invalid arguments are passed in, it will issue an error:: - $ prog.py a b c + $ python prog.py a b c usage: prog.py [-h] [--sum] N [N ...] prog.py: error: argument N: invalid int value: 'a' From 0753e3f2c432b182e4ca0f69e7d9c93c9fd83c6e Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Tue, 8 Oct 2013 08:05:33 +0200 Subject: [PATCH 463/480] Closes #19178: backport entries for "module" and "package" from 3.x glossary. Patch by Berker Peksag. --- Doc/glossary.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 3af49fd0297..5ae957871a3 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -365,6 +365,10 @@ Glossary fraction. Integer division can be forced by using the ``//`` operator instead of the ``/`` operator. See also :term:`__future__`. + importing + The process by which Python code in one module is made available to + Python code in another module. + importer An object that both finds and loads a module; both a :term:`finder` and :term:`loader` object. @@ -509,6 +513,13 @@ Glossary for a member during lookup. See `The Python 2.3 Method Resolution Order `_. + module + An object that serves as an organizational unit of Python code. Modules + have a namespace containing arbitrary Python objects. Modules are loaded + into Python by the process of :term:`importing`. + + See also :term:`package`. + MRO See :term:`method resolution order`. @@ -562,6 +573,11 @@ Glossary (methods). Also the ultimate base class of any :term:`new-style class`. + package + A Python :term:`module` which can contain submodules or recursively, + subpackages. Technically, a package is a Python module with an + ``__path__`` attribute. + parameter A named entity in a :term:`function` (or method) definition that specifies an :term:`argument` (or in some cases, arguments) that the From 61e2fcae0f1d54c610352b75b44656e6e0b67c20 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 8 Oct 2013 21:07:21 +0300 Subject: [PATCH 464/480] Issue #18037: Do not escape '\u' and '\U' in raw strings. --- Lib/lib2to3/fixes/fix_unicode.py | 3 +-- Lib/lib2to3/tests/test_fixers.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/lib2to3/fixes/fix_unicode.py b/Lib/lib2to3/fixes/fix_unicode.py index 922486b5d85..2d776f61051 100644 --- a/Lib/lib2to3/fixes/fix_unicode.py +++ b/Lib/lib2to3/fixes/fix_unicode.py @@ -28,8 +28,7 @@ def transform(self, node, results): return new elif node.type == token.STRING: val = node.value - if (not self.unicode_literals and val[0] in u'rR\'"' and - u'\\' in val): + if not self.unicode_literals and val[0] in u'\'"' and u'\\' in val: val = ur'\\'.join([ v.replace(u'\\u', ur'\\u').replace(u'\\U', ur'\\U') for v in val.split(ur'\\') diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index 5f283a83598..6801c1703bb 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -2830,7 +2830,7 @@ def test_native_literal_escape_u(self): self.check(b, a) b = """r'\\\\\\u20ac\\U0001d121\\\\u20ac'""" - a = """r'\\\\\\\\u20ac\\\\U0001d121\\\\u20ac'""" + a = """r'\\\\\\u20ac\\U0001d121\\\\u20ac'""" self.check(b, a) def test_bytes_literal_escape_u(self): From 4201c2d26e2a38c15278a7635bd0e8fed9047b10 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Tue, 8 Oct 2013 19:50:26 +0200 Subject: [PATCH 465/480] Closes #13867: remove untrue comment about PyWeakref_Check(). --- Include/weakrefobject.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index e46aecf3858..e7c0eae5398 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -49,9 +49,6 @@ PyAPI_DATA(PyTypeObject) _PyWeakref_CallableProxyType; ((Py_TYPE(op) == &_PyWeakref_ProxyType) || \ (Py_TYPE(op) == &_PyWeakref_CallableProxyType)) -/* This macro calls PyWeakref_CheckRef() last since that can involve a - function call; this makes it more likely that the function call - will be avoided. */ #define PyWeakref_Check(op) \ (PyWeakref_CheckRef(op) || PyWeakref_CheckProxy(op)) From f10bbf528489a838f695004082155f486c75263d Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Tue, 8 Oct 2013 21:43:39 +0200 Subject: [PATCH 466/480] Fix wrong link. --- Doc/library/exceptions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index c12e874a39f..ad75ed5fe0d 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -346,7 +346,7 @@ The following exceptions are the exceptions that are actually raised. it has another type (such as a string), the object's value is printed and the exit status is one. - Instances have an attribute :attr:`code` which is set to the proposed exit + Instances have an attribute :attr:`!code` which is set to the proposed exit status or error message (defaulting to ``None``). Also, this exception derives directly from :exc:`BaseException` and not :exc:`StandardError`, since it is not technically an error. From fa5f89d109d3dd0487f3daa3bf7d18c225613165 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Tue, 8 Oct 2013 21:54:37 +0200 Subject: [PATCH 467/480] Minor grammar fix. --- Doc/howto/argparse.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 3c2b592990d..f110a55bba7 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -63,7 +63,7 @@ A few concepts we can learn from the four commands: * That's a snippet of the help text. It's very useful in that you can come across a program you have never used before, and can figure out - how it works simply by reading it's help text. + how it works simply by reading its help text. The basics From a81d9e77942dddc0ca49cb170b194ff2b107c300 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Tue, 8 Oct 2013 20:55:51 -0500 Subject: [PATCH 468/480] Issue 19158: a rare race in BoundedSemaphore could allow .release() too often. (grafted from e06edc0c7a4951327f0c95ebeebccba6879a6063) --- Lib/test/test_threading.py | 17 +++++++++++++++++ Lib/threading.py | 8 +++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 5a765f32083..ac0b4bc701d 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -466,6 +466,23 @@ def test_is_alive_after_fork(self): finally: sys.setcheckinterval(old_interval) + def test_BoundedSemaphore_limit(self): + # BoundedSemaphore should raise ValueError if released too often. + for limit in range(1, 10): + bs = threading.BoundedSemaphore(limit) + threads = [threading.Thread(target=bs.acquire) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + threads = [threading.Thread(target=bs.release) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertRaises(ValueError, bs.release) class ThreadJoinOnShutdown(BaseTestCase): diff --git a/Lib/threading.py b/Lib/threading.py index 72c83197fd1..167f8c7fe31 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -531,9 +531,11 @@ def release(self): raise a ValueError. """ - if self._Semaphore__value >= self._initial_value: - raise ValueError("Semaphore released too many times") - return _Semaphore.release(self) + with self._cond: + if self._value >= self._initial_value: + raise ValueError("Semaphore released too many times") + self._value += 1 + self._cond.notify() def Event(*args, **kwargs): From c2130b92a25a605eb11bfaa2cee9838fb8b717ca Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Tue, 8 Oct 2013 21:51:06 -0500 Subject: [PATCH 469/480] LOL - 2.7 requires very different code. Annoying ;-) --- Lib/threading.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/threading.py b/Lib/threading.py index 167f8c7fe31..e81471bbb00 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -531,11 +531,11 @@ def release(self): raise a ValueError. """ - with self._cond: - if self._value >= self._initial_value: + with self._Semaphore__cond: + if self._Semaphore__value >= self._initial_value: raise ValueError("Semaphore released too many times") - self._value += 1 - self._cond.notify() + self._Semaphore__value += 1 + self._Semaphore__cond.notify() def Event(*args, **kwargs): From 65002494adec627122c24b66d709ba8707d13bdb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 9 Oct 2013 09:54:32 +0300 Subject: [PATCH 470/480] Issue #19193: Improved cross-references in the tutorial. --- Doc/tutorial/classes.rst | 9 +++--- Doc/tutorial/stdlib2.rst | 62 +++++++++++++++++++++------------------- 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 2cc3610c220..eadeeacf321 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -609,7 +609,7 @@ will do nicely:: A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For instance, if you have a function that formats some data from a file object, you -can define a class with methods :meth:`read` and :meth:`readline` that get the +can define a class with methods :meth:`read` and :meth:`!readline` that get the data from a string buffer instead, and pass it as an argument. .. (Unfortunately, this technique has its limitations: a class can't define @@ -693,9 +693,10 @@ using a :keyword:`for` statement:: This style of access is clear, concise, and convenient. The use of iterators pervades and unifies Python. Behind the scenes, the :keyword:`for` statement calls :func:`iter` on the container object. The function returns an iterator -object that defines the method :meth:`next` which accesses elements in the -container one at a time. When there are no more elements, :meth:`next` raises a -:exc:`StopIteration` exception which tells the :keyword:`for` loop to terminate. +object that defines the method :meth:`~iterator.next` which accesses elements +in the container one at a time. When there are no more elements, +:meth:`~iterator.next` raises a :exc:`StopIteration` exception which tells the +:keyword:`for` loop to terminate. This example shows how it all works:: >>> s = 'abc' diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 1b0e856a67d..65fb093a1db 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -71,9 +71,9 @@ formatting numbers with group separators:: Templating ========== -The :mod:`string` module includes a versatile :class:`Template` class with a -simplified syntax suitable for editing by end-users. This allows users to -customize their applications without having to alter the application. +The :mod:`string` module includes a versatile :class:`~string.Template` class +with a simplified syntax suitable for editing by end-users. This allows users +to customize their applications without having to alter the application. The format uses placeholder names formed by ``$`` with valid Python identifiers (alphanumeric characters and underscores). Surrounding the placeholder with @@ -85,11 +85,11 @@ spaces. Writing ``$$`` creates a single escaped ``$``:: >>> t.substitute(village='Nottingham', cause='the ditch fund') 'Nottinghamfolk send $10 to the ditch fund.' -The :meth:`substitute` method raises a :exc:`KeyError` when a placeholder is not -supplied in a dictionary or a keyword argument. For mail-merge style -applications, user supplied data may be incomplete and the -:meth:`safe_substitute` method may be more appropriate --- it will leave -placeholders unchanged if data is missing:: +The :meth:`~string.Template.substitute` method raises a :exc:`KeyError` when a +placeholder is not supplied in a dictionary or a keyword argument. For +mail-merge style applications, user supplied data may be incomplete and the +:meth:`~string.Template.safe_substitute` method may be more appropriate --- +it will leave placeholders unchanged if data is missing:: >>> t = Template('Return the $item to $owner.') >>> d = dict(item='unladen swallow') @@ -132,8 +132,9 @@ templates for XML files, plain text reports, and HTML web reports. Working with Binary Data Record Layouts ======================================= -The :mod:`struct` module provides :func:`pack` and :func:`unpack` functions for -working with variable length binary record formats. The following example shows +The :mod:`struct` module provides :func:`~struct.pack` and +:func:`~struct.unpack` functions for working with variable length binary +record formats. The following example shows how to loop through header information in a ZIP file without using the :mod:`zipfile` module. Pack codes ``"H"`` and ``"I"`` represent two and four byte unsigned numbers respectively. The ``"<"`` indicates that they are @@ -229,8 +230,9 @@ This produces the following output: By default, informational and debugging messages are suppressed and the output is sent to standard error. Other output options include routing messages through email, datagrams, sockets, or to an HTTP Server. New filters can select -different routing based on message priority: :const:`DEBUG`, :const:`INFO`, -:const:`WARNING`, :const:`ERROR`, and :const:`CRITICAL`. +different routing based on message priority: :const:`~logging.DEBUG`, +:const:`~logging.INFO`, :const:`~logging.WARNING`, :const:`~logging.ERROR`, +and :const:`~logging.CRITICAL`. The logging system can be configured directly from Python or can be loaded from a user editable configuration file for customized logging without altering the @@ -287,11 +289,11 @@ Many data structure needs can be met with the built-in list type. However, sometimes there is a need for alternative implementations with different performance trade-offs. -The :mod:`array` module provides an :class:`array()` object that is like a list -that stores only homogeneous data and stores it more compactly. The following -example shows an array of numbers stored as two byte unsigned binary numbers -(typecode ``"H"``) rather than the usual 16 bytes per entry for regular lists of -Python int objects:: +The :mod:`array` module provides an :class:`~array.array()` object that is like +a list that stores only homogeneous data and stores it more compactly. The +following example shows an array of numbers stored as two byte unsigned binary +numbers (typecode ``"H"``) rather than the usual 16 bytes per entry for regular +lists of Python int objects:: >>> from array import array >>> a = array('H', [4000, 10, 700, 22222]) @@ -300,10 +302,10 @@ Python int objects:: >>> a[1:3] array('H', [10, 700]) -The :mod:`collections` module provides a :class:`deque()` object that is like a -list with faster appends and pops from the left side but slower lookups in the -middle. These objects are well suited for implementing queues and breadth first -tree searches:: +The :mod:`collections` module provides a :class:`~collections.deque()` object +that is like a list with faster appends and pops from the left side but slower +lookups in the middle. These objects are well suited for implementing queues +and breadth first tree searches:: >>> from collections import deque >>> d = deque(["task1", "task2", "task3"]) @@ -349,8 +351,8 @@ not want to run a full list sort:: Decimal Floating Point Arithmetic ================================= -The :mod:`decimal` module offers a :class:`Decimal` datatype for decimal -floating point arithmetic. Compared to the built-in :class:`float` +The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for +decimal floating point arithmetic. Compared to the built-in :class:`float` implementation of binary floating point, the class is especially helpful for * financial applications and other uses which require exact decimal @@ -374,13 +376,15 @@ becomes significant if the results are rounded to the nearest cent:: >>> round(.70 * 1.05, 2) # same calculation with floats 0.73 -The :class:`Decimal` result keeps a trailing zero, automatically inferring four -place significance from multiplicands with two place significance. Decimal -reproduces mathematics as done by hand and avoids issues that can arise when -binary floating point cannot exactly represent decimal quantities. +The :class:`~decimal.Decimal` result keeps a trailing zero, automatically +inferring four place significance from multiplicands with two place +significance. Decimal reproduces mathematics as done by hand and avoids +issues that can arise when binary floating point cannot exactly represent +decimal quantities. -Exact representation enables the :class:`Decimal` class to perform modulo -calculations and equality tests that are unsuitable for binary floating point:: +Exact representation enables the :class:`~decimal.Decimal` class to perform +modulo calculations and equality tests that are unsuitable for binary floating +point:: >>> Decimal('1.00') % Decimal('.10') Decimal('0.00') From efe075d58cb985177a46ef825df2f654fa114166 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 9 Oct 2013 13:25:21 +0300 Subject: [PATCH 471/480] Issue 19195: Improved cross-references in C API documentation. --- Doc/c-api/codec.rst | 8 ++++---- Doc/c-api/file.rst | 3 ++- Doc/c-api/object.rst | 20 ++++++++++---------- Doc/c-api/set.rst | 2 +- Doc/c-api/typeobj.rst | 8 ++++---- Doc/c-api/unicode.rst | 6 +++--- Doc/c-api/veryhigh.rst | 2 +- Doc/extending/building.rst | 5 +++-- Doc/extending/extending.rst | 6 +++--- Doc/extending/newtypes.rst | 2 +- 10 files changed, 32 insertions(+), 30 deletions(-) diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst index 8207ae044d7..83252afbb7f 100644 --- a/Doc/c-api/codec.rst +++ b/Doc/c-api/codec.rst @@ -52,19 +52,19 @@ and *NULL* returned. .. c:function:: PyObject* PyCodec_IncrementalEncoder(const char *encoding, const char *errors) - Get an :class:`IncrementalEncoder` object for the given *encoding*. + Get an :class:`~codecs.IncrementalEncoder` object for the given *encoding*. .. c:function:: PyObject* PyCodec_IncrementalDecoder(const char *encoding, const char *errors) - Get an :class:`IncrementalDecoder` object for the given *encoding*. + Get an :class:`~codecs.IncrementalDecoder` object for the given *encoding*. .. c:function:: PyObject* PyCodec_StreamReader(const char *encoding, PyObject *stream, const char *errors) - Get a :class:`StreamReader` factory function for the given *encoding*. + Get a :class:`~codecs.StreamReader` factory function for the given *encoding*. .. c:function:: PyObject* PyCodec_StreamWriter(const char *encoding, PyObject *stream, const char *errors) - Get a :class:`StreamWriter` factory function for the given *encoding*. + Get a :class:`~codecs.StreamWriter` factory function for the given *encoding*. Registry API for Unicode encoding error handlers diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index 20a7a60cddc..bbd422368f9 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -111,7 +111,8 @@ change in future releases of Python. .. index:: single: EOFError (built-in exception) Equivalent to ``p.readline([n])``, this function reads one line from the - object *p*. *p* may be a file object or any object with a :meth:`readline` + object *p*. *p* may be a file object or any object with a + :meth:`~io.IOBase.readline` method. If *n* is ``0``, exactly one line is read, regardless of the length of the line. If *n* is greater than ``0``, no more than *n* bytes will be read from the file; a partial line can be returned. In both cases, an empty string diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a02326f35e9..50b83e90d22 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -47,8 +47,8 @@ Object Protocol Generic attribute getter function that is meant to be put into a type object's ``tp_getattro`` slot. It looks for a descriptor in the dictionary of classes in the object's MRO as well as an attribute in the object's - :attr:`__dict__` (if present). As outlined in :ref:`descriptors`, data - descriptors take preference over instance attributes, while non-data + :attr:`~object.__dict__` (if present). As outlined in :ref:`descriptors`, + data descriptors take preference over instance attributes, while non-data descriptors don't. Otherwise, an :exc:`AttributeError` is raised. @@ -72,8 +72,8 @@ Object Protocol object's ``tp_setattro`` slot. It looks for a data descriptor in the dictionary of classes in the object's MRO, and if found it takes preference over setting the attribute in the instance dictionary. Otherwise, the - attribute is set in the object's :attr:`__dict__` (if present). Otherwise, - an :exc:`AttributeError` is raised and ``-1`` is returned. + attribute is set in the object's :attr:`~object.__dict__` (if present). + Otherwise, an :exc:`AttributeError` is raised and ``-1`` is returned. .. c:function:: int PyObject_DelAttr(PyObject *o, PyObject *attr_name) @@ -180,9 +180,9 @@ Object Protocol be done against every entry in *cls*. The result will be ``1`` when at least one of the checks returns ``1``, otherwise it will be ``0``. If *inst* is not a class instance and *cls* is neither a type object, nor a class object, nor a - tuple, *inst* must have a :attr:`__class__` attribute --- the class relationship - of the value of that attribute with *cls* will be used to determine the result - of this function. + tuple, *inst* must have a :attr:`~instance.__class__` attribute --- the + class relationship of the value of that attribute with *cls* will be used + to determine the result of this function. .. versionadded:: 2.1 @@ -196,9 +196,9 @@ of. If :class:`A` and :class:`B` are class objects, :class:`B` is a subclass of either is not a class object, a more general mechanism is used to determine the class relationship of the two objects. When testing if *B* is a subclass of *A*, if *A* is *B*, :c:func:`PyObject_IsSubclass` returns true. If *A* and *B* -are different objects, *B*'s :attr:`__bases__` attribute is searched in a -depth-first fashion for *A* --- the presence of the :attr:`__bases__` attribute -is considered sufficient for this determination. +are different objects, *B*'s :attr:`~class.__bases__` attribute is searched in +a depth-first fashion for *A* --- the presence of the :attr:`~class.__bases__` +attribute is considered sufficient for this determination. .. c:function:: int PyObject_IsSubclass(PyObject *derived, PyObject *cls) diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 41c4af4fee1..258530b1957 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -156,7 +156,7 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return 1 if found and removed, 0 if not found (no action taken), and -1 if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`PyExc_SystemError` if *set* is an not an instance of :class:`set` or its subtype. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index bc596cca8fa..54f704a7d42 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -603,7 +603,7 @@ The following three fields only exist if the On the other hand, even if you know a member can never be part of a cycle, as a debugging aid you may want to visit it anyway just so the :mod:`gc` module's - :func:`get_referents` function will include it. + :func:`~gc.get_referents` function will include it. Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to :c:func:`local_traverse` to have these specific names; don't name them just @@ -733,7 +733,7 @@ set. reference list head than the base type. Since the list head is always found via :c:member:`~PyTypeObject.tp_weaklistoffset`, this should not be a problem. - When a type defined by a class statement has no :attr:`__slots__` declaration, + When a type defined by a class statement has no :attr:`~object.__slots__` declaration, and none of its base types are weakly referenceable, the type is made weakly referenceable by adding a weak reference list head slot to the instance layout and setting the :c:member:`~PyTypeObject.tp_weaklistoffset` of that slot's offset. @@ -927,7 +927,7 @@ The next fields, up to and including :c:member:`~PyTypeObject.tp_weaklist`, only dictionary at a difference offset than the base type. Since the dictionary is always found via :c:member:`~PyTypeObject.tp_dictoffset`, this should not be a problem. - When a type defined by a class statement has no :attr:`__slots__` declaration, + When a type defined by a class statement has no :attr:`~object.__slots__` declaration, and none of its base types has an instance variable dictionary, a dictionary slot is added to the instance layout and the :c:member:`~PyTypeObject.tp_dictoffset` is set to that slot's offset. @@ -935,7 +935,7 @@ The next fields, up to and including :c:member:`~PyTypeObject.tp_weaklist`, only When a type defined by a class statement has a :attr:`__slots__` declaration, the type inherits its :c:member:`~PyTypeObject.tp_dictoffset` from its base type. - (Adding a slot named :attr:`__dict__` to the :attr:`__slots__` declaration does + (Adding a slot named :attr:`~object.__dict__` to the :attr:`__slots__` declaration does not have the expected effect, it just causes confusion. Maybe this should be added as a feature just like :attr:`__weakref__` though.) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index fd4fe233f14..ddeaaa240b0 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -455,9 +455,9 @@ These are the generic codec APIs: Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* and return a Python string object. *encoding* and *errors* have the same meaning as the parameters - of the same name in the Unicode :meth:`encode` method. The codec to be used is - looked up using the Python codec registry. Return *NULL* if an exception was - raised by the codec. + of the same name in the Unicode :meth:`~unicode.encode` method. The codec + to be used is looked up using the Python codec registry. Return *NULL* if + an exception was raised by the codec. .. versionchanged:: 2.5 This function used an :c:type:`int` type for *size*. This might require diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 4ce3b03efbe..61076655774 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -265,7 +265,7 @@ the same library that the Python runtime is using. frame *f* is executed, interpreting bytecode and executing calls as needed. The additional *throwflag* parameter can mostly be ignored - if true, then it causes an exception to immediately be thrown; this is used for the - :meth:`throw` methods of generator objects. + :meth:`~generator.throw` methods of generator objects. .. c:function:: int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index f4d95b2b0e1..08b0cc2ceda 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -58,8 +58,9 @@ distutils; this section explains building extension modules only. It is common to pre-compute arguments to :func:`setup`, to better structure the driver script. In the example above, the\ ``ext_modules`` argument to :func:`setup` is a list of extension modules, each of which is an instance of -the :class:`Extension`. In the example, the instance defines an extension named -``demo`` which is build by compiling a single source file, :file:`demo.c`. +the :class:`~distutils.extension.Extension`. In the example, the instance +defines an extension named ``demo`` which is build by compiling a single source +file, :file:`demo.c`. In many cases, building an extension is more complex, since additional preprocessor defines and libraries may be needed. This is demonstrated in the diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 97c6d03e174..8e8c3ab575b 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -845,9 +845,9 @@ the cycle itself. The cycle detector is able to detect garbage cycles and can reclaim them so long as there are no finalizers implemented in Python (:meth:`__del__` methods). When there are such finalizers, the detector exposes the cycles through the -:mod:`gc` module (specifically, the -``garbage`` variable in that module). The :mod:`gc` module also exposes a way -to run the detector (the :func:`collect` function), as well as configuration +:mod:`gc` module (specifically, the :attr:`~gc.garbage` variable in that module). +The :mod:`gc` module also exposes a way to run the detector (the +:func:`~gc.collect` function), as well as configuration interfaces and the ability to disable the detector at runtime. The cycle detector is considered an optional component; though it is included by default, it can be disabled at build time using the :option:`--without-cycle-gc` option diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 427821335c4..d76aa2499b2 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -152,7 +152,7 @@ This is so that Python knows how much memory to allocate when you call If you want your type to be subclassable from Python, and your type has the same :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple inheritance. A Python subclass of your type will have to list your type first - in its :attr:`__bases__`, or else it will not be able to call your type's + in its :attr:`~class.__bases__`, or else it will not be able to call your type's :meth:`__new__` method without getting an error. You can avoid this problem by ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its base type does. Most of the time, this will be true anyway, because either your From 91d532d559f030dc83db1c23965e5a22a5a2cfbf Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 9 Oct 2013 14:02:14 +0300 Subject: [PATCH 472/480] Issue #19190: Improve cross-references in builtin types and functions documentation. --- Doc/glossary.rst | 5 ++-- Doc/library/functions.rst | 43 ++++++++++++++++++----------------- Doc/library/stdtypes.rst | 24 +++++++++---------- Doc/reference/datamodel.rst | 36 +++++++++++++++-------------- Doc/reference/expressions.rst | 10 ++++---- 5 files changed, 62 insertions(+), 56 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 5ae957871a3..b5e81717832 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -563,7 +563,7 @@ Glossary new-style class Any class which inherits from :class:`object`. This includes all built-in types like :class:`list` and :class:`dict`. Only new-style classes can - use Python's newer, versatile features like :attr:`__slots__`, + use Python's newer, versatile features like :attr:`~object.__slots__`, descriptors, properties, and :meth:`__getattribute__`. More information can be found in :ref:`newstyle`. @@ -703,7 +703,8 @@ Glossary type The type of a Python object determines what kind of object it is; every object has a type. An object's type is accessible as its - :attr:`__class__` attribute or can be retrieved with ``type(obj)``. + :attr:`~instance.__class__` attribute or can be retrieved with + ``type(obj)``. universal newlines A manner of interpreting text streams in which all of the following are diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index d40452adbbc..aabf2346bb5 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -226,8 +226,8 @@ available. They are listed here in alphabetical order. Future statements are specified by bits which can be bitwise ORed together to specify multiple statements. The bitfield required to specify a given feature - can be found as the :attr:`compiler_flag` attribute on the :class:`_Feature` - instance in the :mod:`__future__` module. + can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on + the :class:`~__future__._Feature` instance in the :mod:`__future__` module. This function raises :exc:`SyntaxError` if the compiled source is invalid, and :exc:`TypeError` if the source contains null bytes. @@ -701,7 +701,7 @@ available. They are listed here in alphabetical order. One useful application of the second form of :func:`iter` is to read lines of a file until a certain line is reached. The following example reads a file - until the :meth:`readline` method returns an empty string:: + until the :meth:`~io.TextIOBase.readline` method returns an empty string:: with open('mydata.txt') as fp: for line in iter(fp.readline, ''): @@ -1013,10 +1013,10 @@ available. They are listed here in alphabetical order. turns the :meth:`voltage` method into a "getter" for a read-only attribute with the same name. - A property object has :attr:`getter`, :attr:`setter`, and :attr:`deleter` - methods usable as decorators that create a copy of the property with the - corresponding accessor function set to the decorated function. This is - best explained with an example:: + A property object has :attr:`~property.getter`, :attr:`~property.setter`, + and :attr:`~property.deleter` methods usable as decorators that create a + copy of the property with the corresponding accessor function set to the + decorated function. This is best explained with an example:: class C(object): def __init__(self): @@ -1259,13 +1259,13 @@ available. They are listed here in alphabetical order. Return a :term:`slice` object representing the set of indices specified by ``range(start, stop, step)``. The *start* and *step* arguments default to - ``None``. Slice objects have read-only data attributes :attr:`start`, - :attr:`stop` and :attr:`step` which merely return the argument values (or their - default). They have no other explicit functionality; however they are used by - Numerical Python and other third party extensions. Slice objects are also - generated when extended indexing syntax is used. For example: - ``a[start:stop:step]`` or ``a[start:stop, i]``. See :func:`itertools.islice` - for an alternate version that returns an iterator. + ``None``. Slice objects have read-only data attributes :attr:`~slice.start`, + :attr:`~slice.stop` and :attr:`~slice.step` which merely return the argument + values (or their default). They have no other explicit functionality; + however they are used by Numerical Python and other third party extensions. + Slice objects are also generated when extended indexing syntax is used. For + example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See + :func:`itertools.islice` for an alternate version that returns an iterator. .. function:: sorted(iterable[, cmp[, key[, reverse]]]) @@ -1370,9 +1370,10 @@ available. They are listed here in alphabetical order. been overridden in a class. The search order is same as that used by :func:`getattr` except that the *type* itself is skipped. - The :attr:`__mro__` attribute of the *type* lists the method resolution - search order used by both :func:`getattr` and :func:`super`. The attribute - is dynamic and can change whenever the inheritance hierarchy is updated. + The :attr:`~class.__mro__` attribute of the *type* lists the method + resolution search order used by both :func:`getattr` and :func:`super`. The + attribute is dynamic and can change whenever the inheritance hierarchy is + updated. If the second argument is omitted, the super object returned is unbound. If the second argument is an object, ``isinstance(obj, type)`` must be true. If @@ -1446,10 +1447,10 @@ available. They are listed here in alphabetical order. With three arguments, return a new type object. This is essentially a dynamic form of the :keyword:`class` statement. The *name* string is the - class name and becomes the :attr:`__name__` attribute; the *bases* tuple - itemizes the base classes and becomes the :attr:`__bases__` attribute; + class name and becomes the :attr:`~class.__name__` attribute; the *bases* tuple + itemizes the base classes and becomes the :attr:`~class.__bases__` attribute; and the *dict* dictionary is the namespace containing definitions for class - body and becomes the :attr:`__dict__` attribute. For example, the + body and becomes the :attr:`~object.__dict__` attribute. For example, the following two statements create identical :class:`type` objects: >>> class X(object): @@ -1513,7 +1514,7 @@ available. They are listed here in alphabetical order. .. function:: vars([object]) - Return the :attr:`__dict__` attribute for a module, class, instance, + Return the :attr:`~object.__dict__` attribute for a module, class, instance, or any other object with a :attr:`__dict__` attribute. Objects such as modules and instances have an updateable :attr:`__dict__` diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 756339dfad7..cd7b683769d 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1734,11 +1734,11 @@ other sequence-like behavior. There are currently two built-in set types, :class:`set` and :class:`frozenset`. The :class:`set` type is mutable --- the contents can be changed using methods -like :meth:`add` and :meth:`remove`. Since it is mutable, it has no hash value -and cannot be used as either a dictionary key or as an element of another set. -The :class:`frozenset` type is immutable and :term:`hashable` --- its contents -cannot be altered after it is created; it can therefore be used as a dictionary -key or as an element of another set. +like :meth:`~set.add` and :meth:`~set.remove`. Since it is mutable, it has no +hash value and cannot be used as either a dictionary key or as an element of +another set. The :class:`frozenset` type is immutable and :term:`hashable` --- +its contents cannot be altered after it is created; it can therefore be used as +a dictionary key or as an element of another set. As of Python 2.7, non-empty sets (not frozensets) can be created by placing a comma-separated list of elements within braces, for example: ``{'jack', @@ -2828,12 +2828,12 @@ statement is not, strictly speaking, an operation on a module object; ``import foo`` does not require a module object named *foo* to exist, rather it requires an (external) *definition* for a module named *foo* somewhere.) -A special attribute of every module is :attr:`__dict__`. This is the dictionary -containing the module's symbol table. Modifying this dictionary will actually -change the module's symbol table, but direct assignment to the :attr:`__dict__` -attribute is not possible (you can write ``m.__dict__['a'] = 1``, which defines -``m.a`` to be ``1``, but you can't write ``m.__dict__ = {}``). Modifying -:attr:`__dict__` directly is not recommended. +A special attribute of every module is :attr:`~object.__dict__`. This is the +dictionary containing the module's symbol table. Modifying this dictionary will +actually change the module's symbol table, but direct assignment to the +:attr:`__dict__` attribute is not possible (you can write +``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but you can't write +``m.__dict__ = {}``). Modifying :attr:`__dict__` directly is not recommended. Modules built into the interpreter are written like this: ````. If loaded from a file, they are written as `` Date: Wed, 9 Oct 2013 14:09:02 +0300 Subject: [PATCH 473/480] #19196: Improved cross-references in distutils documentation. --- Doc/distutils/apiref.rst | 17 +++++++++-------- Doc/distutils/setupscript.rst | 16 +++++++++------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index 480feabfba6..f121b54f485 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -989,8 +989,9 @@ directories. simply the list of all files under *src*, with the names changed to be under *dst*. - *preserve_mode* and *preserve_times* are the same as for :func:`copy_file` in - :mod:`distutils.file_util`; note that they only apply to regular files, not to + *preserve_mode* and *preserve_times* are the same as for + :func:`distutils.file_util.copy_file`; note that they only apply to + regular files, not to directories. If *preserve_symlinks* is true, symlinks will be copied as symlinks (on platforms that support them!); otherwise (the default), the destination of the symlink will be copied. *update* and *verbose* are the same @@ -1172,8 +1173,8 @@ other utility module. or :exc:`OSError`) exception object. Handles Python 1.5.1 and later styles, and does what it can to deal with exception objects that don't have a filename (which happens when the error is due to a two-file operation, such as - :func:`rename` or :func:`link`). Returns the error message as a string - prefixed with *prefix*. + :func:`~os.rename` or :func:`~os.link`). Returns the error message as a + string prefixed with *prefix*. .. function:: split_quoted(s) @@ -1257,8 +1258,8 @@ other utility module. built/installed/distributed -This module provides the :class:`Distribution` class, which represents the -module distribution being built/installed/distributed. +This module provides the :class:`~distutils.core.Distribution` class, which +represents the module distribution being built/installed/distributed. :mod:`distutils.extension` --- The Extension class @@ -1706,8 +1707,8 @@ This module supplies the abstract base class :class:`Command`. options, is the :meth:`run` method, which must also be implemented by every command class. - The class constructor takes a single argument *dist*, a :class:`Distribution` - instance. + The class constructor takes a single argument *dist*, a + :class:`~distutils.core.Distribution` instance. Creating a new Distutils command diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst index 533af01f501..1d13a9cce94 100644 --- a/Doc/distutils/setupscript.rst +++ b/Doc/distutils/setupscript.rst @@ -139,7 +139,8 @@ directories, libraries to link with, etc.). All of this is done through another keyword argument to :func:`setup`, the :option:`ext_modules` option. :option:`ext_modules` is just a list of -:class:`Extension` instances, each of which describes a single extension module. +:class:`~distutils.core.Extension` instances, each of which describes a +single extension module. Suppose your distribution includes a single extension, called :mod:`foo` and implemented by :file:`foo.c`. If no additional instructions to the compiler/linker are needed, describing this extension is quite simple:: @@ -165,8 +166,8 @@ following sections. Extension names and packages ---------------------------- -The first argument to the :class:`Extension` constructor is always the name of -the extension, including any package names. For example, :: +The first argument to the :class:`~distutils.core.Extension` constructor is +always the name of the extension, including any package names. For example, :: Extension('foo', ['src/foo1.c', 'src/foo2.c']) @@ -196,7 +197,8 @@ will compile :file:`foo.c` to the extension :mod:`pkg.foo`, and :file:`bar.c` to Extension source files ---------------------- -The second argument to the :class:`Extension` constructor is a list of source +The second argument to the :class:`~distutils.core.Extension` constructor is +a list of source files. Since the Distutils currently only support C, C++, and Objective-C extensions, these are normally C/C++/Objective-C source files. (Be sure to use appropriate extensions to distinguish C++\ source files: :file:`.cc` and @@ -232,9 +234,9 @@ linked into the executable. Preprocessor options -------------------- -Three optional arguments to :class:`Extension` will help if you need to specify -include directories to search or preprocessor macros to define/undefine: -``include_dirs``, ``define_macros``, and ``undef_macros``. +Three optional arguments to :class:`~distutils.core.Extension` will help if +you need to specify include directories to search or preprocessor macros to +define/undefine: ``include_dirs``, ``define_macros``, and ``undef_macros``. For example, if your extension requires header files in the :file:`include` directory under your distribution root, use the ``include_dirs`` option:: From a9c5f66a7a2808dd0af6f9860b36a7b52692659a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 9 Oct 2013 14:20:06 +0300 Subject: [PATCH 474/480] #19194: Improved cross-references in the fcntl module documentation. --- Doc/library/fcntl.rst | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 40ae08be3e3..cf4ee9fac7e 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -1,6 +1,5 @@ - -:mod:`fcntl` --- The :func:`fcntl` and :func:`ioctl` system calls -================================================================= +:mod:`fcntl` --- The ``fcntl`` and ``ioctl`` system calls +========================================================= .. module:: fcntl :platform: Unix @@ -18,7 +17,7 @@ interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. All functions in this module take a file descriptor *fd* as their first argument. This can be an integer file descriptor, such as returned by ``sys.stdin.fileno()``, or a file object, such as ``sys.stdin`` itself, which -provides a :meth:`fileno` which returns a genuine file descriptor. +provides a :meth:`~io.IOBase.fileno` which returns a genuine file descriptor. The module defines the following functions: @@ -26,7 +25,8 @@ The module defines the following functions: .. function:: fcntl(fd, op[, arg]) Perform the requested operation on file descriptor *fd* (file objects providing - a :meth:`fileno` method are accepted as well). The operation is defined by *op* + a :meth:`~io.IOBase.fileno` method are accepted as well). The operation is + defined by *op* and is operating system dependent. These codes are also found in the :mod:`fcntl` module. The argument *arg* is optional, and defaults to the integer value ``0``. When present, it can either be an integer value, or a string. @@ -46,7 +46,7 @@ The module defines the following functions: .. function:: ioctl(fd, op[, arg[, mutate_flag]]) - This function is identical to the :func:`fcntl` function, except that the + This function is identical to the :func:`~fcntl.fcntl` function, except that the operations are typically defined in the library module :mod:`termios` and the argument handling is even more complicated. @@ -56,7 +56,8 @@ The module defines the following functions: integer ``0``), an object supporting the read-only buffer interface (most likely a plain Python string) or an object supporting the read-write buffer interface. - In all but the last case, behaviour is as for the :func:`fcntl` function. + In all but the last case, behaviour is as for the :func:`~fcntl.fcntl` + function. If a mutable buffer is passed, then the behaviour is determined by the value of the *mutate_flag* parameter. @@ -95,16 +96,16 @@ The module defines the following functions: .. function:: flock(fd, op) Perform the lock operation *op* on file descriptor *fd* (file objects providing - a :meth:`fileno` method are accepted as well). See the Unix manual + a :meth:`~io.IOBase.fileno` method are accepted as well). See the Unix manual :manpage:`flock(2)` for details. (On some systems, this function is emulated using :c:func:`fcntl`.) .. function:: lockf(fd, operation, [length, [start, [whence]]]) - This is essentially a wrapper around the :func:`fcntl` locking calls. *fd* is - the file descriptor of the file to lock or unlock, and *operation* is one of the - following values: + This is essentially a wrapper around the :func:`~fcntl.fcntl` locking calls. + *fd* is the file descriptor of the file to lock or unlock, and *operation* + is one of the following values: * :const:`LOCK_UN` -- unlock * :const:`LOCK_SH` -- acquire a shared lock @@ -119,13 +120,13 @@ The module defines the following functions: systems, :const:`LOCK_EX` can only be used if the file descriptor refers to a file opened for writing. - *length* is the number of bytes to lock, *start* is the byte offset at which the - lock starts, relative to *whence*, and *whence* is as with :func:`fileobj.seek`, - specifically: + *length* is the number of bytes to lock, *start* is the byte offset at + which the lock starts, relative to *whence*, and *whence* is as with + :func:`io.IOBase.seek`, specifically: - * :const:`0` -- relative to the start of the file (:const:`SEEK_SET`) - * :const:`1` -- relative to the current buffer position (:const:`SEEK_CUR`) - * :const:`2` -- relative to the end of the file (:const:`SEEK_END`) + * :const:`0` -- relative to the start of the file (:data:`os.SEEK_SET`) + * :const:`1` -- relative to the current buffer position (:data:`os.SEEK_CUR`) + * :const:`2` -- relative to the end of the file (:data:`os.SEEK_END`) The default for *start* is 0, which means to start at the beginning of the file. The default for *length* is 0 which means to lock to the end of the file. The @@ -150,7 +151,8 @@ lay-out for the *lockdata* variable is system dependent --- therefore using the .. seealso:: Module :mod:`os` - If the locking flags :const:`O_SHLOCK` and :const:`O_EXLOCK` are present - in the :mod:`os` module (on BSD only), the :func:`os.open` function - provides an alternative to the :func:`lockf` and :func:`flock` functions. + If the locking flags :data:`~os.O_SHLOCK` and :data:`~os.O_EXLOCK` are + present in the :mod:`os` module (on BSD only), the :func:`os.open` + function provides an alternative to the :func:`lockf` and :func:`flock` + functions. From 16c8c6a28f99fee3d0855e1b104fc909d78aabbd Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Wed, 9 Oct 2013 13:25:11 -0500 Subject: [PATCH 475/480] Issue 19158: a rare race in BoundedSemaphore could allow .release() too often. --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 9aad84fc177..144e80176ed 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #19158: a rare race in BoundedSemaphore could allow .release() too + often. + - Issue #18037: 2to3 now escapes '\u' and '\U' in native strings. - Issue #19137: The pprint module now correctly formats empty set and frozenset From ac59bb61325b8dd4c2d32946f14b27dc01544223 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 9 Oct 2013 22:39:11 -0700 Subject: [PATCH 476/480] Issue #19005: Fix documentation for PyIter_Next(). --- Doc/c-api/iter.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index 88ac0c1b2ff..8d1567c7b34 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -7,7 +7,7 @@ Iterator Protocol .. versionadded:: 2.2 -There are only a couple of functions specifically for working with iterators. +There are two functions specifically for working with iterators. .. c:function:: int PyIter_Check(PyObject *o) @@ -17,11 +17,10 @@ There are only a couple of functions specifically for working with iterators. .. c:function:: PyObject* PyIter_Next(PyObject *o) - Return the next value from the iteration *o*. If the object is an iterator, - this retrieves the next value from the iteration, and returns *NULL* with no - exception set if there are no remaining items. If the object is not an - iterator, :exc:`TypeError` is raised, or if there is an error in retrieving the - item, returns *NULL* and passes along the exception. + Return the next value from the iteration *o*. The object must be an iterator + (it is up to the caller to check this). If there are no remaining values, + returns *NULL* with no exception set. If an error occurs while retrieving + the item, returns *NULL* and passes along the exception. To write a loop which iterates over an iterator, the C code should look something like this:: From 6b45edcddf6c48ded9177ee6a8efa2cb8b5fa6ba Mon Sep 17 00:00:00 2001 From: bendmorris Date: Fri, 11 Oct 2013 15:13:20 -0400 Subject: [PATCH 477/480] Initial commit. Re-creating repository as old repo was a fork of a bad mirror. --- README => PYTHON-README | 0 README.md | 99 +++++++++++ Static.make | 35 ++++ add_builtins.py | 352 ++++++++++++++++++++++++++++++++++++++++ static_freeze.py | 97 +++++++++++ 5 files changed, 583 insertions(+) rename README => PYTHON-README (100%) create mode 100644 README.md create mode 100644 Static.make create mode 100644 add_builtins.py create mode 100755 static_freeze.py diff --git a/README b/PYTHON-README similarity index 100% rename from README rename to PYTHON-README diff --git a/README.md b/README.md new file mode 100644 index 00000000000..62502b29e0f --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +Static-Python +============= + +This is a fork of the official Python hg repository with additional tools to +enable building Python for static linking. + +You may be wondering, "why would you want to do that? After all, [static linking +is evil](http://www.akkadia.org/drepper/no_static_linking.html)." Here are a few +possible reasons: + + * To run Python programs on other machines without requiring that they install + Python. + + * To run Python programs on other machines without requiring that they have + the same versions of the same libraries installed that you do. + + * Because the major binary distribution tools for Python (cx_Freeze, bbfreeze, + py2exe, and py2app) ship in a way that the Python source code can be + trivially derived (unzip the archive of .pyc files and decompile them.) For + proprietary or security-conscious applications, this is unacceptable. + + +Usage +===== + +Building Static Python +---------------------- + +To build a static Python executable and library, check out the appropriate branch +(either 2.7, 3.3, or master) and run the following command: + + make -f Static.make + +This will create an executable called `python` in the working directory, and a +static library, `libpythonX.X.a`, in the install/lib directory. You can confirm +that this executable is not dependent on any shared libraries using `ldd python` +which should report that python is not a dynamic executable. However, by default +this executable's functionality will be very limited - it won't even be able to +access most modules from the Python standard library. + +In order to make this Python interpreter truly standalone (not dependent on +installed Python modules), you can designate Python modules to be compiled as +builtins, which will be statically linked into the Python interpreter. +Static.make generates a file in Modules/Setup which needs to be edited to +specify these new builtin modules. + +You can automatically add builtins when building Static Python by passing +BUILTINS and/or SCRIPT variables to Static.make, e.g.: + + make -f Static.make BUILTINS="math zipfile zlib" SCRIPT="/path/to/script.py" + +Each module listed in the BUILTINS variable will be added, if possible. SCRIPT +can be used to specify the path to a Python script. This script will be scanned +for dependencies using modulefinder, and all dependencies will be added as +builtins if possible (not the script itself - it should be compiled using +static_freeze.py and linked to the resulting static library.) Finally, if the +DFLAG variable is set to "-d", all dependencies of all modules will be +automatically added as well (this will usually include many modules, and you may +not really need them all.) + +(If you previously built Static Python, you should `make -f Static.make clean` +first. Also, this step requires an existing Python installation, preferably of +the same version you're building, so you may need to build and install Python +the normal way first before building it statically..) + +Adding builtins can also be done manually by editing Modules/Setup. Add lines to +the end of the file in the format: + + module_name module.c ... + +These .c files can be generated from .py files using Cython: + + cython module.py + +(This is done automatically using add_builtins.py, the script called by +Static.make when BUILTINS or SCRIPT are supplied.) + +Packages are not currently supported. I'm working on an automatic solution for +compiling and including packages. Stay tuned. + + +Compile a standalone executable +------------------------------- + +Once you've compiled a static Python library, you can turn a Python script into +a standalone executable using the static_freeze.py script. + + Tools/static_freeze/static_freeze.py test.py libpython2.7.a + +This will generate an executable called "test" in the working directory, which +is not dependant on any shared libraries, Python modules, or the Python +interpreter! + + +Acknowledgements +================ + +Major thanks to Gabriel Jacobo who pioneered this method: + diff --git a/Static.make b/Static.make new file mode 100644 index 00000000000..93bbbcf266a --- /dev/null +++ b/Static.make @@ -0,0 +1,35 @@ +.PHONY: all setup clean + +all: python +setup: Modules/Setup + +PYTHON=python +PREFIX=$(shell pwd)/install +CONF_ARGS= +MAKE_ARGS= +#BUILTINS=array cmath math _struct time _operator _testcapi _random _collections _heapq itertools _functools _elementtree _pickle _datetime _bisect unicodedata atexit _weakref +BUILTINS=atexit +SCRIPT= +DFLAG= +CPPFLAGS= +LDFLAGS= +INCLUDE=-I/usr/include + +Modules/Setup: Modules/Setup.dist add_builtins.py + sed -e 's/#\*shared\*/\*static\*/g' Modules/Setup.dist \ + > Modules/Setup + [ -d Modules/extras ] || mkdir Modules/extras + $(PYTHON) add_builtins.py $(BUILTINS) $(DFLAG) -s $(SCRIPT) + +Makefile: Modules/Setup + [ -d $(PREFIX) ] || mkdir $(PREFIX) + ./configure LDFLAGS="-Wl,-no-export-dynamic -static-libgcc -static $(LDFLAGS) $(INCLUDE)" \ + CPPFLAGS="-I/usr/lib -static -fPIC $(CPPLAGS) $(INCLUDE)" LINKFORSHARED=" " \ + DYNLOADFILE="dynload_stub.o" -disable-shared \ + -prefix="$(PREFIX)" $(CONF_ARGS) + +python: Modules/Setup Makefile + make $(MAKE_ARGS) + +clean: + rm -f Makefile Modules/Setup \ No newline at end of file diff --git a/add_builtins.py b/add_builtins.py new file mode 100644 index 00000000000..fae609d49a3 --- /dev/null +++ b/add_builtins.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python +'''Add a set of modules to the list of static Python builtins. + +Usage: python add_builtins.py [-s /path/to/script] [module_name] ... +''' +from modulefinder import ModuleFinder +import sys +import os +import os.path as op +import re +import imp +import shutil +import types +import importlib +from Cython.Build import cythonize +import Cython + + +# module definition lines in Modules/Setup look like this +module_def = re.compile('^[A-Za-z_\.]+ .+\.c') +# put extra builtins in here +extra_module_dir = op.join('Modules', 'extras') +if not op.exists(extra_module_dir): + os.makedirs(extra_module_dir) +# file endings that can be compiled by cython +compileable_exts = ('.c', '.cpp', 'module.c', 'module.cpp') + +def add_builtins(names, script=None, exclude=None, path=None, auto_add_deps=False, src_dirs=None): + if path is None: + paths = ['lib'] + sys.path + elif isinstance(path, basestring): + paths = [path, 'lib'] + sys.path + else: + paths = path + + if src_dirs is None: src_dirs = {} + + # if called with a script, find their dependencies and re-run + to_add = set(names) + if script: + module_dir = op.split(script)[0] + paths = [module_dir] + paths + finder = ModuleFinder(path=paths) + finder.run_script(script) + to_add.update(finder.modules.keys()) + return add_builtins(list(to_add), script=None, exclude=exclude, + path=paths, auto_add_deps=False, src_dirs=None) + + if auto_add_deps: + for name in names: + try: + f, module_path, _ = imp.find_module(name, paths) + except KeyboardInterrupt: raise + except: continue + + if any([module_path.endswith(x) for x in ('.py', '.pyc')]): + finder = ModuleFinder(path=paths) + finder.run_script(module_path) + to_add.update(finder.modules.keys()) + + + with open('Modules/Setup', 'r') as setup_file: + lines = [str(x).rstrip('\n') for x in setup_file.readlines()] + + # don't add sys (it's already builtin) or anything explicitly excluded + added = {'sys'} + if exclude: + added.update(exclude) + + + # check each module to see if a commented line is present in Modules/Setup, + # and uncomment + for n, line in enumerate(lines): + line = line.strip() + if not line: continue + + for name in to_add: + if line.startswith('#%s ' % name): + lines[n] = line.lstrip('#') + to_add.remove(name) + added.add(name) + print('** Added %s' % name) + break + + # keep track of uncommented module names in Modules/Setup + if module_def.match(line): + module_name = line.split()[0] + pkg = False + try: + f, module_path, _ = imp.find_module(module_name, paths) + if f is None: pkg = True + except: pkg = False + + if not pkg: + added.add(module_name) + print('** Found existing builtin %s' % module_name) + + # don't try to re-add existing builtins + to_add = set.difference(to_add, added) + + for name in list(to_add): + if name in added: continue + + new_lines = [] + + try: + f, module_path, _ = imp.find_module(name, paths) + except ImportError: + if '.' in name: f = None + else: + raise Exception("** Couldn't find module %s" % name) + continue + + # see if the target file already exists in Modules + search_paths = [op.join(*(search_dir + (name+x,))) + for x in compileable_exts + for search_dir in ( + (), + (name,), + ('extras', name), + ) + if op.exists(op.join(*(('Modules',) + search_dir + (name+x,)))) + ] if f else [] + + if search_paths: + module_file = search_paths[0] + print('** Added %s' % module_file) + else: + # import the target module using this python installation, + # and check the corresponding file + + if f is None: + # add package + pkg = name + print("*** Scanning package %s..." % pkg) + + try: + p = importlib.import_module(pkg) + except: + continue + + def get_submodules(x, yielded=None): + if not yielded: yielded = set() + yield x + yielded.add(x) + for member in dir(x): + member = getattr(x, member) + if isinstance(member, types.ModuleType) and not member in yielded: + for y in get_submodules(member, yielded): + yield y + yielded.add(y) + + submodules = get_submodules(p) + + for submodule in submodules: + name = submodule.__name__ + + sys.stdout.write("** Adding module %s in package %s..." % (name, pkg)) + sys.stdout.flush() + + try: + add = add_module(name, added, paths, src_dirs) + if add: new_lines += add + except Exception as e: + print(e) + + print('done.') + + else: + # add standalone module + sys.stdout.write('** Adding %s...' % name) + sys.stdout.flush() + + try: + add = add_module(name, added, paths, src_dirs, module_path=module_path) + if add: new_lines += add + except Exception as e: + print(e) + + print('done.') + + if new_lines: lines += new_lines + + with open('Modules/Setup', 'w') as setup_file: + setup_file.write('\n'.join(lines)) + + + +def add_module(name, added, paths, src_dirs, module_path=None): + if name in added: + return + + added.add(name) + pkg = '.' in name + opts = '' + + if not module_path: + try: module_path = importlib.import_module(name).__file__ + except: return + + # if it's a .pyc file, hope the original python source is right next to it! + if module_path.endswith('.pyc'): + if op.exists(module_path[:-1]): + module_path = module_path[:-1] + else: + raise Exception('Lone .pyc file %s' % module_path) + + module_dir, module_file = op.split(module_path) + + # copy the file to the Modules/extras directory + dest_dir = extra_module_dir + if pkg: + dest_dir = op.join(dest_dir, name.split('.')[0]) + if not op.exists(dest_dir): os.makedirs(dest_dir) + + + # if it's a shared library, try to find the original C or C++ + # source file to compile into a static library; otherwise, + # there's nothing we can do here + if module_file.endswith('.so'): + done = False + module_dirs = [] + for k, v in src_dirs.items(): + # if user specified a src directory for this package, + # include it in the module search path + if name.startswith(k): + module = name[len(k):].lstrip('.') + if '.' in module: + v = op.join(v, *module.split('.')) + module_dirs += [v] + module_dirs += [module_dir] + for search_dir in module_dirs + paths: + for compiled_module in ('.'.join(module_file.split('.')[:-1]) + ext + for ext in compileable_exts): + if op.exists(op.join(search_dir, compiled_module)): + dest_file = compiled_module + if pkg: + dest_file = '__'.join(name.split('.')) + '.' + dest_file.split('.')[-1] + + dest_path = op.join(dest_dir, dest_file) + + if not op.exists(dest_path): + shutil.copy(op.join(search_dir, compiled_module), dest_path) + + modile_dir = search_dir + module_file = dest_file + opts += ' -I%s' % op.abspath(search_dir) + done = True; break + + if done: break + + if not any([module_file.endswith(ext) for ext in compileable_exts]): + #raise Exception("Couldn't find C source file for %s" % module_file) + return + + module_path = op.join(module_dir, module_file) + + else: + # copy the file to the Modules/extras directory + dest_file = module_file + if pkg: + dest_file = '__'.join(name.split('.')) + '.' + dest_file.split('.')[-1] + + dest_path = op.join(dest_dir, dest_file) + opts = '' + + if not op.exists(dest_path): + shutil.copy(module_path, dest_path) + + + # if the file ends in .py or .pyx, try to compile with Cython + if any([module_file.endswith(x) for x in ('.py', '.pyx')]): + dest_file = '.'.join(dest_file.split('.')[:-1]) + '.c' + + if op.exists(op.join(dest_dir, dest_file)): + module_file = dest_file + else: + try: + cythonize(dest_path) + module_file = dest_file + dest_path = op.join(dest_dir, dest_file) + if pkg: + # correct module name in Cython-generated C file + wrong_name = '.'.join(dest_file.split('.')[:-1]) + print wrong_name, name, dest_path + + with open(dest_path, 'r') as input_file: + with open(dest_path + '2', 'w') as output_file: + for line in input_file: + line = line.replace('"%s"' % wrong_name, '"%s"' % name) + output_file.write(line) + + os.remove(dest_path) + shutil.move(dest_path + '2', dest_path) + + except KeyboardInterrupt: raise + except: + os.remove(op.join(dest_dir, dest_file)) + raise Exception('Cython failed to compile %s' % dest_path) + + if any([module_file.endswith(ext) for ext in compileable_exts]): + # if there's a directory called Modules/{module} or + # Modules/extras/{module}, include that directory when compiling + for inc_dir in (op.join('Modules', name), op.join(dest_dir, name)): + if op.exists(inc_dir) and op.isdir(inc_dir): + opts += ' -I%s' % op.abspath(inc_dir) + + if pkg: + module_file = op.join('extras', + name.split('.')[0], + '.'.join(module_file.split('.')[:-1]).replace( + '.', '__') + '.' + module_file.split('.')[-1] + ) + else: + module_file = op.join('extras', module_file) + + # add a line to Modules/Setup + return ['%s %s%s' % (name.replace('.', '__'), module_file, opts)] + else: + raise Exception('Unknown file: %s' % module_file) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser() + + parser.add_argument('module', nargs='*', help='names of modules to be added as builtins') + parser.add_argument('-s', '--script', nargs='?', default=None, + help='add all dependencies of this script as builtins') + parser.add_argument('-e', '--exclude', nargs='?', default=None, + help='comma-separated list of modules to be excluded') + parser.add_argument('-p', '--path', nargs='?', default=None, + help='add this path to the module search path') + parser.add_argument('-d', '--deps', action='store_true', + help='when adding a module, automatically add all of its dependencies') + parser.add_argument('--src', nargs='?', default=None, + help='list of source package locations for shared libraries, e.g. `pkg1:/path/to/src,pkg2:/path/to/src`') + + args = parser.parse_args() + + src_dirs = {pkg.split(':')[0]:pkg.split(':')[1] + for pkg in args.src.split(',') + } if args.src else None + + add_builtins(args.module, + script=args.script, + exclude=args.exclude.split(',') if args.exclude else None, + path=args.path, + auto_add_deps=args.deps, + src_dirs=src_dirs + ) diff --git a/static_freeze.py b/static_freeze.py new file mode 100755 index 00000000000..e555bfd4650 --- /dev/null +++ b/static_freeze.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +'''This script generates, runs, and then deletes a Makefile which will compile a +Python script into a standalone executable which is statically linked to the +Python runtime. + +To use this script, first generate a static Python library (see the parent +directory's README.md). + +Usage: + + python static_freeze.py test_file.py /path/to/static/libpythonX.X.a + +Any additional arguments will be passed to the generated Makefile, +e.g. PYTHON=python3 +''' + +import sys +import os +from subprocess import call +major_version = sys.version_info[0] + + +make_template = '''# Makefile for creating our standalone Cython program +PYTHON=./python +CYTHON=cython +PYVERSION=$(shell $(PYTHON) -c "import sys; print(sys.version[:3])") + +#INCDIR=$(shell $(PYTHON) -c "from distutils import sysconfig; print(sysconfig.get_python_inc())") +INCDIR=Include/ +PLATINCDIR=$(shell $(PYTHON) -c "from distutils import sysconfig; print(sysconfig.get_python_inc(plat_specific=True))") +LIBDIR1=$(shell $(PYTHON) -c "from distutils import sysconfig; print(sysconfig.get_config_var('LIBDIR'))") +LIBDIR2=%library_dir% +PYLIB=%library_name% + +CC=$(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('CC'))") +LINKCC=$(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('LINKCC'))") +LIBS=$(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('LIBS'))") +SYSLIBS= $(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('SYSLIBS'))") + +INCLUDE= + +%name%: %name%.o %library_dir%/lib%library_name%.a + $(LINKCC) -o $@ $^ -L$() -L$(LIBDIR1) -L$(LIBDIR2) -l$(PYLIB) $(LIBS) $(SYSLIBS) $(INCLUDE) + +%name%.o: %name%.c + $(CC) -c $^ -I$(INCDIR) -I$(PLATINCDIR) $(INCLUDE) + +%name%.c: %filename% + @$(CYTHON) -%major% --embed %filename% + +all: %name% + +clean: + rm -f %name% %name%.c %name%.o +''' + +def freeze(filename, library, make_args=None): + if make_args is None: make_args = [] + + name = '.'.join(filename.split('.')[:-1]) + library_dir, library_name = os.path.split(os.path.abspath(library)) + library_name = '.'.join((library.split('/')[-1][3:]).split('.')[:-1]) + + template = make_template + # generate makefile + for a, b in ( + ('%name%', name), + ('%filename%', filename), + ('%library_dir%', library_dir), + ('%library_name%', library_name), + ('%major%', str(major_version)), + ): + template = template.replace(a, b) + + with open(filename + '.make', 'wb') as make_file: + make_file.write(bytes(template, 'utf8')) + + # call make + call(['make', '-f', '%s.make' % filename] + make_args) + + # delete makefile + os.remove(filename + '.make') + + +if __name__ == '__main__': + import sys + + try: script_file = sys.argv[1] + except: raise Exception('No script specified') + + try: lib_path = sys.argv[2] + except: raise Exception('Path to Python runtime not specified') + + try: make_args = sys.argv[3:] + except: make_args = [] + + freeze(script_file, lib_path, make_args=make_args) \ No newline at end of file From c9ea3a0d10a487de43136b54854f3b42155fbe7e Mon Sep 17 00:00:00 2001 From: bendmorris Date: Fri, 11 Oct 2013 15:30:10 -0400 Subject: [PATCH 478/480] Friendlier failures in static_freeze.py --- static_freeze.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/static_freeze.py b/static_freeze.py index e555bfd4650..2a3c2b2f7b0 100755 --- a/static_freeze.py +++ b/static_freeze.py @@ -83,15 +83,28 @@ def freeze(filename, library, make_args=None): if __name__ == '__main__': - import sys + def fail(message): + print(__doc__) + print(message) + sys.exit() - try: script_file = sys.argv[1] - except: raise Exception('No script specified') - - try: lib_path = sys.argv[2] - except: raise Exception('Path to Python runtime not specified') + try: + script_file = sys.argv[1] + assert os.path.exists(script_file) + except IndexError: + fail('ERROR: No script specified') + except AssertionError: + fail('ERROR: Script not found') + + try: + lib_path = sys.argv[2] + assert os.path.exists(lib_path) + except IndexError: + fail('ERROR: Path to Python runtime not specified') + except AssertionError: + fail('ERROR: Python runtime not found') try: make_args = sys.argv[3:] - except: make_args = [] + except IndexError: make_args = [] freeze(script_file, lib_path, make_args=make_args) \ No newline at end of file From 88f67d6fd698126c6b5daa7e37fc81af64916f54 Mon Sep 17 00:00:00 2001 From: bendmorris Date: Fri, 11 Oct 2013 21:09:35 -0400 Subject: [PATCH 479/480] Fixing issue with package adding - now handling __init__.py correctly. --- Static.make | 4 ++-- add_builtins.py | 43 +++++++++++++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 16 deletions(-) mode change 100644 => 100755 add_builtins.py diff --git a/Static.make b/Static.make index 93bbbcf266a..92addd0beb4 100644 --- a/Static.make +++ b/Static.make @@ -7,8 +7,8 @@ PYTHON=python PREFIX=$(shell pwd)/install CONF_ARGS= MAKE_ARGS= -#BUILTINS=array cmath math _struct time _operator _testcapi _random _collections _heapq itertools _functools _elementtree _pickle _datetime _bisect unicodedata atexit _weakref -BUILTINS=atexit +BUILTINS= +override BUILTINS+= array cmath math _struct time _operator _testcapi _random _collections _heapq itertools _functools _elementtree _pickle _datetime _bisect unicodedata atexit _weakref datetime SCRIPT= DFLAG= CPPFLAGS= diff --git a/add_builtins.py b/add_builtins.py old mode 100644 new mode 100755 index fae609d49a3..162ed6759db --- a/add_builtins.py +++ b/add_builtins.py @@ -1,6 +1,13 @@ #!/usr/bin/env python '''Add a set of modules to the list of static Python builtins. +This will modify the Modules/Setup file, which is created by the Static.make +Makefile. When running `make -f Static.make`, this script will be run +automatically using any arguments passed to that Makefile. If you want to add +modules manually, you can run `make -f Static.make setup` which will build +Modules/Setup and allow you to edit it or run this script one or more times. +When finished, run `make -f Static.make` to build the static interpreter. + Usage: python add_builtins.py [-s /path/to/script] [module_name] ... ''' from modulefinder import ModuleFinder @@ -13,7 +20,6 @@ import types import importlib from Cython.Build import cythonize -import Cython # module definition lines in Modules/Setup look like this @@ -22,14 +28,17 @@ extra_module_dir = op.join('Modules', 'extras') if not op.exists(extra_module_dir): os.makedirs(extra_module_dir) -# file endings that can be compiled by cython +# file endings that can be cythonized +cythonizeable_exts = ('.py', '.pyx') +# file endings that can be compiled compileable_exts = ('.c', '.cpp', 'module.c', 'module.cpp') -def add_builtins(names, script=None, exclude=None, path=None, auto_add_deps=False, src_dirs=None): +def add_builtins(names, script=None, exclude=None, path=None, + auto_add_deps=False, src_dirs=None, test=False): if path is None: - paths = ['lib'] + sys.path + paths = ['Lib'] + sys.path elif isinstance(path, basestring): - paths = [path, 'lib'] + sys.path + paths = [path, 'Lib'] + sys.path else: paths = path @@ -162,7 +171,8 @@ def get_submodules(x, yielded=None): add = add_module(name, added, paths, src_dirs) if add: new_lines += add except Exception as e: - print(e) + print('Failed:', e) + #raise print('done.') @@ -197,12 +207,16 @@ def add_module(name, added, paths, src_dirs, module_path=None): if not module_path: try: module_path = importlib.import_module(name).__file__ except: return + + if op.basename(module_path).startswith('__init__'): + pkg = True # if it's a .pyc file, hope the original python source is right next to it! if module_path.endswith('.pyc'): if op.exists(module_path[:-1]): module_path = module_path[:-1] else: + # TODO: this could possibly be handled by unpyclib, etc. raise Exception('Lone .pyc file %s' % module_path) module_dir, module_file = op.split(module_path) @@ -214,9 +228,8 @@ def add_module(name, added, paths, src_dirs, module_path=None): if not op.exists(dest_dir): os.makedirs(dest_dir) - # if it's a shared library, try to find the original C or C++ - # source file to compile into a static library; otherwise, - # there's nothing we can do here + # if it's a shared library, try to find the original C or C++ source file to + # compile into a static library; otherwise, there's nothing we can do here if module_file.endswith('.so'): done = False module_dirs = [] @@ -250,8 +263,8 @@ def add_module(name, added, paths, src_dirs, module_path=None): if done: break if not any([module_file.endswith(ext) for ext in compileable_exts]): - #raise Exception("Couldn't find C source file for %s" % module_file) - return + raise Exception("Couldn't find C source file for %s" % module_file) + #return module_path = op.join(module_dir, module_file) @@ -269,7 +282,7 @@ def add_module(name, added, paths, src_dirs, module_path=None): # if the file ends in .py or .pyx, try to compile with Cython - if any([module_file.endswith(x) for x in ('.py', '.pyx')]): + if any([module_file.endswith(x) for x in cythonizeable_exts]): dest_file = '.'.join(dest_file.split('.')[:-1]) + '.c' if op.exists(op.join(dest_dir, dest_file)): @@ -282,7 +295,6 @@ def add_module(name, added, paths, src_dirs, module_path=None): if pkg: # correct module name in Cython-generated C file wrong_name = '.'.join(dest_file.split('.')[:-1]) - print wrong_name, name, dest_path with open(dest_path, 'r') as input_file: with open(dest_path + '2', 'w') as output_file: @@ -336,6 +348,8 @@ def add_module(name, added, paths, src_dirs, module_path=None): help='when adding a module, automatically add all of its dependencies') parser.add_argument('--src', nargs='?', default=None, help='list of source package locations for shared libraries, e.g. `pkg1:/path/to/src,pkg2:/path/to/src`') + parser.add_argument('-t', '--test', action='store_true', + help="don't actually add the modules right now, just output their names") args = parser.parse_args() @@ -348,5 +362,6 @@ def add_module(name, added, paths, src_dirs, module_path=None): exclude=args.exclude.split(',') if args.exclude else None, path=args.path, auto_add_deps=args.deps, - src_dirs=src_dirs + src_dirs=src_dirs, + test=args.test ) From f3f531ec0289e73677af628b2a16924495c24bc3 Mon Sep 17 00:00:00 2001 From: bendmorris Date: Sat, 12 Oct 2013 06:06:41 -0400 Subject: [PATCH 480/480] Removing _testcapi from default builtins. --- Static.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Static.make b/Static.make index 92addd0beb4..3dd23aab598 100644 --- a/Static.make +++ b/Static.make @@ -8,7 +8,7 @@ PREFIX=$(shell pwd)/install CONF_ARGS= MAKE_ARGS= BUILTINS= -override BUILTINS+= array cmath math _struct time _operator _testcapi _random _collections _heapq itertools _functools _elementtree _pickle _datetime _bisect unicodedata atexit _weakref datetime +override BUILTINS+= array cmath math _struct time _operator _random _collections _heapq itertools _functools _elementtree _pickle _datetime _bisect unicodedata atexit _weakref datetime SCRIPT= DFLAG= CPPFLAGS=