From 0981cb336b4db6adf6601525ddb8f0d3dfb95bb3 Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Fri, 23 Jan 2015 10:05:08 -0800 Subject: [PATCH 01/34] Map issue kind/priority/status to their bitbucket equivalents Also mark the default_issue_kind as required --- .../bitbucket_issue_converter.py | 38 +++++++++++++++++-- .../bitbucket_issue_converter_test.py | 6 +-- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/googlecode-issues-exporter/bitbucket_issue_converter.py b/googlecode-issues-exporter/bitbucket_issue_converter.py index b718646..d0ea8b2 100644 --- a/googlecode-issues-exporter/bitbucket_issue_converter.py +++ b/googlecode-issues-exporter/bitbucket_issue_converter.py @@ -28,6 +28,38 @@ import issues +def _getKind(kind): + mapping = { + "defect": "bug", + "enhancement": "enhancement", + "task": "task", + "review": "proposal", + "other": "bug", + } + return mapping.get(kind.lower(), "bug") + + +def _getPriority(priority): + mapping = { + "low": "trivial", + "medium": "minor", + "high": "major", + "critical": "critical", + } + return mapping.get(priority.lower(), "minor") + + +def _getStatus(status): + mapping = { + "new": "new", + "fixed": "resolved", + "invalid": "invalid", + "duplicate": "duplicate", + "wontfix": "wontfix", + } + return mapping.get(status.lower(), "new") + + class UserService(issues.UserService): """BitBucket user operations. """ @@ -80,10 +112,10 @@ def CreateIssue(self, googlecode_issue): "content_updated_on": googlecode_issue.GetContentUpdatedOn(), "created_on": googlecode_issue.GetCreatedOn(), "id": googlecode_issue.GetId(), - "kind": googlecode_issue.GetKind(), - "priority": googlecode_issue.GetPriority(), + "kind": _getKind(googlecode_issue.GetKind()), + "priority": _getPriority(googlecode_issue.GetPriority()), "reporter": googlecode_issue.GetAuthor(), - "status": googlecode_issue.GetStatus(), + "status": _getStatus(googlecode_issue.GetStatus()), "title": googlecode_issue.GetTitle(), "updated_on": googlecode_issue.GetUpdatedOn() } diff --git a/googlecode-issues-exporter/bitbucket_issue_converter_test.py b/googlecode-issues-exporter/bitbucket_issue_converter_test.py index a2f5de4..3147173 100644 --- a/googlecode-issues-exporter/bitbucket_issue_converter_test.py +++ b/googlecode-issues-exporter/bitbucket_issue_converter_test.py @@ -68,10 +68,10 @@ def testCreateIssue(self): "content_updated_on": "last month", "created_on": "last year", "id": 1, - "kind": "Defect", - "priority": "Medium", + "kind": "bug", + "priority": "minor", "reporter": None, - "status": "fixed", + "status": "resolved", "title": "issue_title", "updated_on": "last year", } From db7f3fae38a2a8a6c41fb367fe8652abc90fd08b Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Fri, 23 Jan 2015 14:49:20 -0800 Subject: [PATCH 02/34] Do not assume that all issues have a label --- googlecode-issues-exporter/bitbucket_issue_converter.py | 2 +- googlecode-issues-exporter/issues.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/googlecode-issues-exporter/bitbucket_issue_converter.py b/googlecode-issues-exporter/bitbucket_issue_converter.py index d0ea8b2..295c164 100644 --- a/googlecode-issues-exporter/bitbucket_issue_converter.py +++ b/googlecode-issues-exporter/bitbucket_issue_converter.py @@ -208,7 +208,7 @@ def main(args): parser.add_argument("--user_file_path", required=True, help="The path to the file containing a mapping from" "email address to bitbucket username") - parser.add_argument("--default_issue_kind", required=False, + parser.add_argument("--default_issue_kind", required=True, help="A non-null string containing one of the following" "values: bug, enhancement, proposal, task. Defaults to" "bug.") diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index c0fef13..a49ed4c 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -123,7 +123,7 @@ def GetLabels(self): Returns: A list of the labels of this issue. """ - return self._issue["labels"] + return self._issue.get("labels", []) def GetKind(self): """Get the kind from a Google Code issue. From ef479d6b555c95241010aebf8632d694bfda1388 Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Fri, 23 Jan 2015 15:31:37 -0800 Subject: [PATCH 03/34] Limit issue title to 255 characters --- .../bitbucket_issue_converter.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/googlecode-issues-exporter/bitbucket_issue_converter.py b/googlecode-issues-exporter/bitbucket_issue_converter.py index 295c164..4580c4a 100644 --- a/googlecode-issues-exporter/bitbucket_issue_converter.py +++ b/googlecode-issues-exporter/bitbucket_issue_converter.py @@ -60,6 +60,12 @@ def _getStatus(status): return mapping.get(status.lower(), "new") +def _getTitle(title): + if len(title) < 255: + return title + return title[:250] + "[...]" + + class UserService(issues.UserService): """BitBucket user operations. """ @@ -116,7 +122,7 @@ def CreateIssue(self, googlecode_issue): "priority": _getPriority(googlecode_issue.GetPriority()), "reporter": googlecode_issue.GetAuthor(), "status": _getStatus(googlecode_issue.GetStatus()), - "title": googlecode_issue.GetTitle(), + "title": _getTitle(googlecode_issue.GetTitle()), "updated_on": googlecode_issue.GetUpdatedOn() } self._bitbucket_issues.append(bitbucket_issue) @@ -161,8 +167,8 @@ def WriteIssueData(self, default_issue_kind): } with open("db-1.0.json", "w") as issues_file: issues_json = json.dumps(issues_data, sort_keys=True, indent=4, - separators=(",", ": "), ensure_ascii=False) - issues_file.write(unicode(issues_json)) + separators=(",", ": ")) + issues_file.write(issues_json) def ExportIssues(issue_file_path, project_name, From 5eb1818e49740756f4be8ecc1fec57bd5f066f3f Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 12 Feb 2015 13:24:35 -0800 Subject: [PATCH 04/34] Set some flags to be optional. --- googlecode-issues-exporter/bitbucket_issue_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/googlecode-issues-exporter/bitbucket_issue_converter.py b/googlecode-issues-exporter/bitbucket_issue_converter.py index 4580c4a..73cd50c 100644 --- a/googlecode-issues-exporter/bitbucket_issue_converter.py +++ b/googlecode-issues-exporter/bitbucket_issue_converter.py @@ -211,10 +211,10 @@ def main(args): parser.add_argument("--project_name", required=True, help="The name of the Google Code project you wish to" "export") - parser.add_argument("--user_file_path", required=True, + parser.add_argument("--user_file_path", required=False, help="The path to the file containing a mapping from" "email address to bitbucket username") - parser.add_argument("--default_issue_kind", required=True, + parser.add_argument("--default_issue_kind", required=False, help="A non-null string containing one of the following" "values: bug, enhancement, proposal, task. Defaults to" "bug.") From 24f68383aea8ae1f127311fde1d76f9fa59d37e0 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 12 Feb 2015 13:28:46 -0800 Subject: [PATCH 05/34] Add default value for default_issue_kind --- googlecode-issues-exporter/bitbucket_issue_converter.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/googlecode-issues-exporter/bitbucket_issue_converter.py b/googlecode-issues-exporter/bitbucket_issue_converter.py index 73cd50c..508a2a1 100644 --- a/googlecode-issues-exporter/bitbucket_issue_converter.py +++ b/googlecode-issues-exporter/bitbucket_issue_converter.py @@ -217,11 +217,16 @@ def main(args): parser.add_argument("--default_issue_kind", required=False, help="A non-null string containing one of the following" "values: bug, enhancement, proposal, task. Defaults to" - "bug.") + "bug") parser.add_argument("--default_owner_username", required=True, - help="The default issue username") + help="The default issue owner's username") parsed_args, _ = parser.parse_known_args(args) + # Default value. + if not parsed_args.default_issue_kind: + print "Using default issue kind of 'bug'." + parsed_args.default_issue_kind = "bug" + ExportIssues( parsed_args.issue_file_path, parsed_args.project_name, parsed_args.user_file_path, parsed_args.default_issue_kind, From ec307471f338f131c8c479414359623ff8108ad8 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 19 Feb 2015 13:19:02 -0800 Subject: [PATCH 06/34] Rename wiki-to-md to wiki_to_md This makes it possible to include wiki_to_md as a Python module. --- {wiki-to-md => wiki_to_md}/convert-repo.sh | 0 {wiki-to-md => wiki_to_md}/example.md | 0 {wiki-to-md => wiki_to_md}/example.wiki | 0 {wiki-to-md => wiki_to_md}/impl/__init__.py | 0 {wiki-to-md => wiki_to_md}/impl/constants.py | 0 {wiki-to-md => wiki_to_md}/impl/converter.py | 0 {wiki-to-md => wiki_to_md}/impl/formatting_handler.py | 0 {wiki-to-md => wiki_to_md}/impl/pragma_handler.py | 0 {wiki-to-md => wiki_to_md}/wiki2gfm.py | 0 {wiki-to-md => wiki_to_md}/wiki2gfm_test.py | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename {wiki-to-md => wiki_to_md}/convert-repo.sh (100%) rename {wiki-to-md => wiki_to_md}/example.md (100%) rename {wiki-to-md => wiki_to_md}/example.wiki (100%) rename {wiki-to-md => wiki_to_md}/impl/__init__.py (100%) rename {wiki-to-md => wiki_to_md}/impl/constants.py (100%) rename {wiki-to-md => wiki_to_md}/impl/converter.py (100%) rename {wiki-to-md => wiki_to_md}/impl/formatting_handler.py (100%) rename {wiki-to-md => wiki_to_md}/impl/pragma_handler.py (100%) rename {wiki-to-md => wiki_to_md}/wiki2gfm.py (100%) rename {wiki-to-md => wiki_to_md}/wiki2gfm_test.py (100%) diff --git a/wiki-to-md/convert-repo.sh b/wiki_to_md/convert-repo.sh similarity index 100% rename from wiki-to-md/convert-repo.sh rename to wiki_to_md/convert-repo.sh diff --git a/wiki-to-md/example.md b/wiki_to_md/example.md similarity index 100% rename from wiki-to-md/example.md rename to wiki_to_md/example.md diff --git a/wiki-to-md/example.wiki b/wiki_to_md/example.wiki similarity index 100% rename from wiki-to-md/example.wiki rename to wiki_to_md/example.wiki diff --git a/wiki-to-md/impl/__init__.py b/wiki_to_md/impl/__init__.py similarity index 100% rename from wiki-to-md/impl/__init__.py rename to wiki_to_md/impl/__init__.py diff --git a/wiki-to-md/impl/constants.py b/wiki_to_md/impl/constants.py similarity index 100% rename from wiki-to-md/impl/constants.py rename to wiki_to_md/impl/constants.py diff --git a/wiki-to-md/impl/converter.py b/wiki_to_md/impl/converter.py similarity index 100% rename from wiki-to-md/impl/converter.py rename to wiki_to_md/impl/converter.py diff --git a/wiki-to-md/impl/formatting_handler.py b/wiki_to_md/impl/formatting_handler.py similarity index 100% rename from wiki-to-md/impl/formatting_handler.py rename to wiki_to_md/impl/formatting_handler.py diff --git a/wiki-to-md/impl/pragma_handler.py b/wiki_to_md/impl/pragma_handler.py similarity index 100% rename from wiki-to-md/impl/pragma_handler.py rename to wiki_to_md/impl/pragma_handler.py diff --git a/wiki-to-md/wiki2gfm.py b/wiki_to_md/wiki2gfm.py similarity index 100% rename from wiki-to-md/wiki2gfm.py rename to wiki_to_md/wiki2gfm.py diff --git a/wiki-to-md/wiki2gfm_test.py b/wiki_to_md/wiki2gfm_test.py similarity index 100% rename from wiki-to-md/wiki2gfm_test.py rename to wiki_to_md/wiki2gfm_test.py From 0cf0bcf1ca9855afb94924d966c09f309de49661 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 19 Feb 2015 13:29:06 -0800 Subject: [PATCH 07/34] Add explanation header --- wiki_to_md/example.md | 14 ++++++++++++++ wiki_to_md/example.wiki | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/wiki_to_md/example.md b/wiki_to_md/example.md index 3682b00..1890d42 100644 --- a/wiki_to_md/example.md +++ b/wiki_to_md/example.md @@ -1,5 +1,19 @@ # Header1 +This is a test file for verifying the tool's output. You can regenerate +` example.md ` yourself by running: + +``` +python wiki2gfm.py \ + --project=test \ + --wikipages_list="TestPage" \ + --input_file=example.wiki \ + --output_file=example.md +``` + +Note that this is used by {{wiki2gfm\_test.py}}, so changing the +arguments will break unit tests. + (TODO: Add table of contents.) #### Tables (mistmatched header sides) diff --git a/wiki_to_md/example.wiki b/wiki_to_md/example.wiki index 450c130..3c78e1d 100644 --- a/wiki_to_md/example.wiki +++ b/wiki_to_md/example.wiki @@ -3,6 +3,20 @@ #labels Ignored. = Header1 = +This is a test file for verifying the tool's output. You can regenerate +{{{example.md}}} yourself by running: + +{{{ +python wiki2gfm.py \ + --project=test \ + --wikipages_list="TestPage" \ + --input_file=example.wiki \ + --output_file=example.md +}}} + +Note that this is used by {{wiki2gfm_test.py}}, so changing the +arguments will break unit tests. + ==== Tables (mistmatched header sides) == From 2fc95811ac80ea4f2bb8758d49ef8ce032db2c90 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 19 Feb 2015 13:30:51 -0800 Subject: [PATCH 08/34] Remove TODO associated with Google+ links. --- wiki_to_md/example.md | 4 ++-- wiki_to_md/impl/formatting_handler.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/wiki_to_md/example.md b/wiki_to_md/example.md index 1890d42..1f96e47 100644 --- a/wiki_to_md/example.md +++ b/wiki_to_md/example.md @@ -245,7 +245,7 @@ rev111612.
This text will be removed from the rendered page. '> -(TODO: Link to Google+ page.) + @@ -254,7 +254,7 @@ This text will be removed from the rendered page. This text will be removed from the rendered page. '>

-(TODO: Link to Google+ page.)
+


diff --git a/wiki_to_md/impl/formatting_handler.py b/wiki_to_md/impl/formatting_handler.py index 75102c8..c421990 100644 --- a/wiki_to_md/impl/formatting_handler.py +++ b/wiki_to_md/impl/formatting_handler.py @@ -738,15 +738,12 @@ def HandleGPlusOpen(self, input_line, output_stream, unused_params): output_stream: Output Markdown file. unused_params: The parameters for the tag. """ - output = "(TODO: Link to Google+ page.)" self._warning_method( input_line, "A Google+ +1 button was embedded on this page, but GitHub does not " "currently support this. Should it become supported in the future, " "see https://developers.google.com/+/web/+1button/ for more " - "information.\nIt has been replaced with:\n\t{0}" - .format(output)) - self._Write(output, output_stream) + "information.\nIt has been removed.") def HandleGPlusClose(self, unused_input_line, unused_output_stream): """Handle the output for closing a +1 button. From d8be80be43b8c43a6c9a1b01328dc4de801bf480 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 19 Feb 2015 13:34:17 -0800 Subject: [PATCH 09/34] Fix unit tests --- wiki_to_md/wiki2gfm_test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/wiki_to_md/wiki2gfm_test.py b/wiki_to_md/wiki2gfm_test.py index 73eb098..60f98c1 100644 --- a/wiki_to_md/wiki2gfm_test.py +++ b/wiki_to_md/wiki2gfm_test.py @@ -50,6 +50,9 @@ def assertOutput(self, expected_output): """ self.assertEquals(expected_output, self.output.getvalue()) + def assertNoOutput(self, expected_output): + self.assertNotEqual(expected_output, self.output.getvalue()) + def assertWarning(self, warning_contents, occurrences=1): """Assert that a warning was issued containing the given contents. @@ -707,7 +710,7 @@ def testHandleGPlus(self): self.formatting_handler.HandleGPlusOpen(1, self.output, None) self.formatting_handler.HandleGPlusClose(1, self.output) - self.assertOutput("(TODO: Link to Google+ page.)") + self.assertNoOutput("(TODO: Link to Google+ page.)") self.assertWarning("A Google+ +1 button was embedded on this page") def testHandleGPlusInHtml(self): @@ -715,7 +718,7 @@ def testHandleGPlusInHtml(self): self.formatting_handler.HandleGPlusOpen(1, self.output, None) self.formatting_handler.HandleGPlusClose(1, self.output) - self.assertOutput("(TODO: Link to Google+ page.)") + self.assertNoOutput("(TODO: Link to Google+ page.)") self.assertWarning("A Google+ +1 button was embedded on this page") def testHandleComment(self): From cabf2ff70b655aee3328bd2282104ded14ad4142 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 19 Feb 2015 13:37:07 -0800 Subject: [PATCH 10/34] Remove TODO left in output from missing pragma --- wiki_to_md/example.md | 2 +- wiki_to_md/impl/converter.py | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/wiki_to_md/example.md b/wiki_to_md/example.md index 1f96e47..391c976 100644 --- a/wiki_to_md/example.md +++ b/wiki_to_md/example.md @@ -14,7 +14,7 @@ python wiki2gfm.py \ Note that this is used by {{wiki2gfm\_test.py}}, so changing the arguments will break unit tests. -(TODO: Add table of contents.) + #### Tables (mistmatched header sides) diff --git a/wiki_to_md/impl/converter.py b/wiki_to_md/impl/converter.py index 39e26c5..fe3c649 100644 --- a/wiki_to_md/impl/converter.py +++ b/wiki_to_md/impl/converter.py @@ -1134,19 +1134,14 @@ def _HandlePluginWikiToc(self, input_line, match, output_stream): match: Matched text. output_stream: Output Markdown file. """ - output = "(TODO: Add table of contents.)" self._warning_method( input_line, "A table of contents plugin was used for this wiki:\n" "\t{0}\n" "The Gollum wiki system supports table of content generation.\n" "See https://github.com/gollum/gollum/wiki for more information.\n" - "It has been replaced with:\n\t{1}" - .format(match, output)) - self._formatting_handler.HandleEscapedText( - input_line, - output_stream, - output) + "It has been removed." + .format(match)) def _HandlePluginEnd(self, input_line, match, output_stream): """Handle a plugin ending tag. From 551264ba549d332c16f5aab910d10c5240a38ffa Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 19 Feb 2015 15:05:23 -0800 Subject: [PATCH 11/34] Remove wiki/ file prefix. Assume same-dir and .md extension --- wiki_to_md/example.md | 8 ++++---- wiki_to_md/impl/formatting_handler.py | 3 ++- wiki_to_md/wiki2gfm_test.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/wiki_to_md/example.md b/wiki_to_md/example.md index 391c976..7240d88 100644 --- a/wiki_to_md/example.md +++ b/wiki_to_md/example.md @@ -179,9 +179,9 @@ The following is:
==== Links -A LittleLink[?](wiki/TestPage) +A LittleLink[?](TestPage.md) -[TestPage](wiki/TestPage) is linked automatically. +[TestPage](TestPage.md) is linked automatically. http://www.google.com/ @@ -198,9 +198,9 @@ http://www.google.com/ **IN HTML** -A LittleLink? +A LittleLink? -TestPage is linked automatically.
+TestPage is linked automatically.

http://www.google.com/ diff --git a/wiki_to_md/impl/formatting_handler.py b/wiki_to_md/impl/formatting_handler.py index c421990..4eb7671 100644 --- a/wiki_to_md/impl/formatting_handler.py +++ b/wiki_to_md/impl/formatting_handler.py @@ -577,7 +577,8 @@ def HandleWiki(self, input_line, output_stream, target, text): """ # A wiki link is just like a regular link, except under the wiki directory. # At this point we make the text equal to the original target if unset. - self.HandleLink(input_line, output_stream, "wiki/" + target, text or target) + # We do however append ".md", assuming the wiki files now have that extension. + self.HandleLink(input_line, output_stream, target + ".md", text or target) def HandleIssue(self, input_line, output_stream, prefix, issue): """Handle the output for an auto-linked issue. diff --git a/wiki_to_md/wiki2gfm_test.py b/wiki_to_md/wiki2gfm_test.py index 60f98c1..cd3594e 100644 --- a/wiki_to_md/wiki2gfm_test.py +++ b/wiki_to_md/wiki2gfm_test.py @@ -502,14 +502,14 @@ def testHandleImageLinkWithImageDescriptionInHtml(self): def testHandleWiki(self): self.formatting_handler.HandleWiki(1, self.output, "TestPage", "Test Page") - self.assertOutput("[Test Page](wiki/TestPage)") + self.assertOutput("[Test Page](TestPage.md)") self.assertNoWarnings() def testHandleWikiInHtml(self): self.formatting_handler._in_html = 1 self.formatting_handler.HandleWiki(1, self.output, "TestPage", "Test Page") - self.assertOutput("Test Page") + self.assertOutput("Test Page") self.assertWarning("Link markup was used") def testHandleIssue(self): From da90810f712665b782a1bac0fe6c5a367b137690 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 9 Mar 2015 14:17:18 -0700 Subject: [PATCH 12/34] Fix problem with extra spaces around `code` Previously we were inserting extra spaces around text which was using the `code` formatting. When rendered as HTML this isn't noticeable, however the Markdown doesn't look right. --- wiki_to_md/example.md | 32 ++++++++++++++++----------- wiki_to_md/example.wiki | 7 ++++++ wiki_to_md/impl/formatting_handler.py | 2 +- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/wiki_to_md/example.md b/wiki_to_md/example.md index 7240d88..38f497b 100644 --- a/wiki_to_md/example.md +++ b/wiki_to_md/example.md @@ -1,7 +1,7 @@ # Header1 This is a test file for verifying the tool's output. You can regenerate -` example.md ` yourself by running: +`example.md` yourself by running: ``` python wiki2gfm.py \ @@ -26,22 +26,22 @@ Tables: |**Name/Sample** | **Markup** | |:-----------------|:-----------------| -| _italic_ | ` _italic_ ` | -| **bold** | ` *bold* ` | -| ` code ` | `` `code` `` | -| ` code ` | ` {{{code}}} ` | -| superscript | ` ^super^script ` | -| subscript | ` ,,sub,,script ` | -| ~~strikeout~~ | ` ~~strikeout~~ ` | +| _italic_ | `_italic_` | +| **bold** | `*bold*` | +| `code` | ```code``` | +| `code` | `{{{code}}}` | +| superscript | `^super^script` | +| subscript | `,,sub,,script` | +| ~~strikeout~~ | `~~strikeout~~` | You can mix these typefaces in some ways: | **Markup** | **Result** | |:------------------------------------|:----------------------------------| -| ` _*bold* in italics_ ` | _**bold** in italics_ | -| ` *_italics_ in bold* ` | **_italics_ in bold** | -| ` *~~strike~~ works too* ` | **~~strike~~ works too** | -| ` ~~as well as _this_ way round~~ ` | ~~as well as _this_ way round~~ | +| `_*bold* in italics_` | _**bold** in italics_ | +| `*_italics_ in bold*` | **_italics_ in bold** | +| `*~~strike~~ works too*` | **~~strike~~ works too** | +| `~~as well as _this_ way round~~` | ~~as well as _this_ way round~~ | **IN HTML** @@ -286,4 +286,10 @@ this is a var!
## All done! -Hope you enjoyed. \ No newline at end of file +Hope you enjoyed. + +# Regressions + +We were adding extra spaces around tick marks: + +alpha `beta` gamma diff --git a/wiki_to_md/example.wiki b/wiki_to_md/example.wiki index 3c78e1d..d647328 100644 --- a/wiki_to_md/example.wiki +++ b/wiki_to_md/example.wiki @@ -289,3 +289,10 @@ This text will be removed from the rendered page. == All done!== Hope you enjoyed. + += Regressions = + +We were adding extra spaces around tick marks: + +alpha `beta` gamma + diff --git a/wiki_to_md/impl/formatting_handler.py b/wiki_to_md/impl/formatting_handler.py index 4eb7671..37bf769 100644 --- a/wiki_to_md/impl/formatting_handler.py +++ b/wiki_to_md/impl/formatting_handler.py @@ -391,7 +391,7 @@ def HandleInlineCode(self, input_line, output_stream, code): consecutive_ticks = 0 surrounding_ticks = "`" * (max_consecutive_ticks + 1) - self._Write("{0} {1} {0}".format(surrounding_ticks, code), output_stream) + self._Write("{0}{1}{0}".format(surrounding_ticks, code), output_stream) def HandleTableCellBorder(self, input_line, output_stream): """Handle the output for a table cell border. From 442f859ed7dd73c9a4d60ab704fe7e474cdf5ce7 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 11 Mar 2015 13:43:43 -0700 Subject: [PATCH 13/34] Add rate_limit flag to disable throttling --- .../github_issue_converter.py | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/googlecode-issues-exporter/github_issue_converter.py b/googlecode-issues-exporter/github_issue_converter.py index 1b30b82..19cbb5e 100644 --- a/googlecode-issues-exporter/github_issue_converter.py +++ b/googlecode-issues-exporter/github_issue_converter.py @@ -89,10 +89,11 @@ class GitHubService(object): Attributes: github_owner_username: The username of the owner of the repository. github_repo_name: The GitHub repository name. + rate_limit: Whether or not to rate limit API calls. """ def __init__(self, github_owner_username, github_repo_name, - github_oauth_token, http_instance=None): + github_oauth_token, rate_limit, http_instance=None): """Initialize the GitHubService. Args: @@ -105,6 +106,7 @@ def __init__(self, github_owner_username, github_repo_name, self.github_owner_username = github_owner_username self.github_repo_name = github_repo_name self._github_oauth_token = github_oauth_token + self._rate_limit = rate_limit self._http = http_instance if http_instance else httplib2.Http() def _PerformHttpRequest(self, method, url, body="{}", params=None): @@ -162,13 +164,14 @@ def PerformPostRequest(self, url, body): A tuple of an HTTP response (https://developer.github.com/v3/#schema) and its content from the server which is decoded JSON. """ - # Add a delay to all outgoing request to GitHub, as to not trigger their - # anti-abuse mechanism. This is separate from your typical rate limit, and - # only applies to certain API calls (like creating issues). And, alas, the - # exact quota is undocumented. So the value below is simply a guess. See: - # https://developer.github.com/v3/#abuse-rate-limits - req_min = 30 - time.sleep(60 / req_min) + if self._rate_limit: + # Add a delay to all outgoing request to GitHub, as to not trigger their + # anti-abuse mechanism. This is separate from your typical rate limit, and + # only applies to certain API calls (like creating issues). And, alas, the + # exact quota is undocumented. So the value below is simply a guess. See: + # https://developer.github.com/v3/#abuse-rate-limits + req_min = 25 + time.sleep(60 / req_min) return self._PerformHttpRequest("POST", url, body) def PerformPatchRequest(self, url, body): @@ -268,7 +271,6 @@ class IssueService(issues.IssueService): def __init__(self, github_service, comment_delay=COMMENT_DELAY): """Initialize the IssueService. - Args: github_service: The GitHub service. """ @@ -397,11 +399,12 @@ def _GetIssueNumber(self, content): def ExportIssues(github_owner_username, github_repo_name, github_oauth_token, - issue_file_path, project_name, user_file_path): + issue_file_path, project_name, user_file_path, rate_limit): """Exports all issues for a given project. """ github_service = GitHubService( - github_owner_username, github_repo_name, github_oauth_token) + github_owner_username, github_repo_name, github_oauth_token, + rate_limit) issue_service = IssueService(github_service) user_service = UserService(github_service) @@ -449,12 +452,16 @@ def main(args): parser.add_argument("--user_file_path", required=False, help="The path to the file containing a mapping from" "email address to github username.") + parser.add_argument("--rate_limit", required=False, default=True, + help="Rate limit GitHub requests to not run into" + "anti-abuse limits.") parsed_args, _ = parser.parse_known_args(args) ExportIssues( parsed_args.github_owner_username, parsed_args.github_repo_name, parsed_args.github_oauth_token, parsed_args.issue_file_path, - parsed_args.project_name, parsed_args.user_file_path) + parsed_args.project_name, parsed_args.user_file_path, + parsed_args.rate_limit) if __name__ == "__main__": From 798e16a46da4db9457b1e49aa934a776a3ed84c5 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 11 Mar 2015 14:15:07 -0700 Subject: [PATCH 14/34] Fix index error in wiki conversion --- wiki_to_md/impl/formatting_handler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wiki_to_md/impl/formatting_handler.py b/wiki_to_md/impl/formatting_handler.py index 37bf769..facd655 100644 --- a/wiki_to_md/impl/formatting_handler.py +++ b/wiki_to_md/impl/formatting_handler.py @@ -914,6 +914,10 @@ def _HandleHtmlListClose(self, input_line, output_stream): input_line: Current line number being processed. output_stream: Output Markdown file. """ + # Fix index error if list_tags is empty. + if len(self._list_tags) == 0: + self._warning_method(input_line, "HtmlListClose without list_tags?") + self._list_tags = [ { "indent": 0, "kind": "Bulleted list" } ] top_tag = self._list_tags[-1] kind = top_tag["kind"] self._list_tags.pop() From 8ac29fcfacaf09abab2d76c372146fa29a9c22ec Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 11 Mar 2015 16:53:17 -0700 Subject: [PATCH 15/34] Add User-Agent header, fix rate_limiting --- googlecode-issues-exporter/github_issue_converter.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/googlecode-issues-exporter/github_issue_converter.py b/googlecode-issues-exporter/github_issue_converter.py index 19cbb5e..637b085 100644 --- a/googlecode-issues-exporter/github_issue_converter.py +++ b/googlecode-issues-exporter/github_issue_converter.py @@ -126,13 +126,15 @@ def _PerformHttpRequest(self, method, url, body="{}", params=None): A tuple of an HTTP response (https://developer.github.com/v3/#schema) and its content from the server which is decoded JSON. """ + headers = { "User-Agent": "GoogleCodeIssueExporter/1.0" } query = params.copy() if params else {} query["access_token"] = self._github_oauth_token request_url = "%s%s?%s" % (GITHUB_API_URL, url, urllib.urlencode(query)) requests = 0 while requests < MAX_HTTP_REQUESTS: requests += 1 - response, content = self._http.request(request_url, method, body) + response, content = self._http.request(request_url, method, + headers=headers, body=body) if _CheckSuccessful(response): return response, json.loads(content) elif self._RequestLimitReached(): @@ -164,7 +166,7 @@ def PerformPostRequest(self, url, body): A tuple of an HTTP response (https://developer.github.com/v3/#schema) and its content from the server which is decoded JSON. """ - if self._rate_limit: + if self._rate_limit and self._rate_limit in ["True", "true"]: # Add a delay to all outgoing request to GitHub, as to not trigger their # anti-abuse mechanism. This is separate from your typical rate limit, and # only applies to certain API calls (like creating issues). And, alas, the @@ -452,7 +454,7 @@ def main(args): parser.add_argument("--user_file_path", required=False, help="The path to the file containing a mapping from" "email address to github username.") - parser.add_argument("--rate_limit", required=False, default=True, + parser.add_argument("--rate_limit", required=False, default="True", help="Rate limit GitHub requests to not run into" "anti-abuse limits.") parsed_args, _ = parser.parse_known_args(args) From 937ced33b1eb4e7dba8265eeb76f1da7fc0448d1 Mon Sep 17 00:00:00 2001 From: Nicholas Gorski Date: Fri, 13 Mar 2015 12:55:31 -0700 Subject: [PATCH 16/34] Fix test, and format with Unicode strings. --- wiki_to_md/impl/converter.py | 36 ++++++++++---------- wiki_to_md/impl/formatting_handler.py | 48 +++++++++++++-------------- wiki_to_md/wiki2gfm_test.py | 2 +- 3 files changed, 43 insertions(+), 43 deletions(-) diff --git a/wiki_to_md/impl/converter.py b/wiki_to_md/impl/converter.py index fe3c649..bcbda64 100644 --- a/wiki_to_md/impl/converter.py +++ b/wiki_to_md/impl/converter.py @@ -128,7 +128,7 @@ def Convert(self, input_stream, output_stream): self._warning_method( input_line, "Processing completed, but not all lines were processed. " - "Remaining lines: {0}.".format(remaining_lines)) + u"Remaining lines: {0}.".format(remaining_lines)) def _ExtractPragmas(self, input_line, input_lines, output_stream): """Extracts pragmas from a given input. @@ -398,7 +398,7 @@ def _SetCurrentList(self, input_line, indent_pos, list_type, output_stream): else: self._warning_method( input_line, - "Bad list type: '{0}'".format(list_type)) + u"Bad list type: '{0}'".format(list_type)) return True @@ -411,11 +411,11 @@ def _OpenTag(self, input_line, tag, output_stream): output_stream: Output Markdown file. """ handler = getattr( - self._formatting_handler, "Handle{0}Open".format(tag), None) + self._formatting_handler, u"Handle{0}Open".format(tag), None) if handler: handler(input_line, output_stream) else: - self._warning_method(input_line, "Bad open tag: '{0}'".format(tag)) + self._warning_method(input_line, u"Bad open tag: '{0}'".format(tag)) self._open_tags.append(tag) @@ -428,11 +428,11 @@ def _CloseTag(self, input_line, tag, output_stream): output_stream: Output Markdown file. """ handler = getattr( - self._formatting_handler, "Handle{0}Close".format(tag), None) + self._formatting_handler, u"Handle{0}Close".format(tag), None) if handler: handler(input_line, output_stream) else: - self._warning_method(input_line, "Bad close tag: '{0}'".format(tag)) + self._warning_method(input_line, u"Bad close tag: '{0}'".format(tag)) self._open_tags.remove(tag) @@ -509,7 +509,7 @@ def _ProcessMatch(self, input_line, match_regex, line, output_stream): output_stream, match) else: - handler = getattr(self, "_Handle{0}".format(rulename), None) + handler = getattr(self, u"_Handle{0}".format(rulename), None) handler(input_line, match, output_stream) lastpos = fullmatch.end() @@ -936,12 +936,12 @@ def _HandlePlugin(self, input_line, match, output_stream): self._warning_method( input_line, "Unknown plugin was given, outputting " - "as plain text:\n\t{0}".format(match)) + u"as plain text:\n\t{0}".format(match)) # Wiki syntax put this class of error on its own line. self._formatting_handler.HandleEscapedText( input_line, output_stream, - "\n\n{0}\n\n".format(match)) + u"\n\n{0}\n\n".format(match)) # Add plugin and parameters to the stack. if not has_end: @@ -1106,7 +1106,7 @@ def _HandlePluginWikiVideo( self._formatting_handler.HandleEscapedText( input_line, output_stream, - "\n\n{0}\n\n".format(output)) + u"\n\n{0}\n\n".format(output)) else: self._formatting_handler.HandleVideoOpen( input_line, @@ -1124,7 +1124,7 @@ def _HandlePluginWikiVideo( self._formatting_handler.HandleEscapedText( input_line, output_stream, - "\n\n{0}\n\n".format(output)) + u"\n\n{0}\n\n".format(output)) def _HandlePluginWikiToc(self, input_line, match, output_stream): """Handle a plugin tag for a wiki table of contents. @@ -1186,24 +1186,24 @@ def _HandlePluginEnd(self, input_line, match, output_stream): self._warning_method( input_line, "Unknown but matching plugin end was given, outputting " - "as plain text:\n\t{0}".format(match)) + u"as plain text:\n\t{0}".format(match)) # Wiki syntax put this class of error on its own line. self._formatting_handler.HandleEscapedText( input_line, output_stream, - "\n\n{0}\n\n".format(match)) + u"\n\n{0}\n\n".format(match)) else: self._warning_method( input_line, "Unknown/unmatched plugin end was given, outputting " - "as plain text with errors:\n\t{0}".format(match)) + u"as plain text with errors:\n\t{0}".format(match)) # Wiki syntax put this class of error on its own line, # with a prefix error message, and did not display the tag namespace. tag_without_ns = plugin_id.split(":", 1)[-1] self._formatting_handler.HandleEscapedText( input_line, output_stream, - "\n\nUnknown end tag for \n\n".format(tag_without_ns)) + u"\n\nUnknown end tag for \n\n".format(tag_without_ns)) def _HandleVariable(self, input_line, match, output_stream): """Handle a variable. @@ -1246,16 +1246,16 @@ def _HandleVariable(self, input_line, match, output_stream): if self._project: output = self._project instructions = ("It has been replaced with static text containing the " - "name of the project:\n\t{0}".format(self._project)) + u"name of the project:\n\t{0}".format(self._project)) else: output = "(TODO: Replace with project name.)" instructions = ("Because no project name was specified, the text has " - "been replaced with:\n\t{0}".format(output)) + u"been replaced with:\n\t{0}".format(output)) # Not defined anywhere, just treat as regular text. if not output: # Add surrounding %% back on. - output = "%%{0}%%".format(match) + output = u"%%{0}%%".format(match) self._formatting_handler.HandleEscapedText( input_line, diff --git a/wiki_to_md/impl/formatting_handler.py b/wiki_to_md/impl/formatting_handler.py index facd655..04c4763 100644 --- a/wiki_to_md/impl/formatting_handler.py +++ b/wiki_to_md/impl/formatting_handler.py @@ -115,7 +115,7 @@ def HandleHeaderOpen(self, input_line, output_stream, header_level): header_level: The header level. """ if self._in_html: - tag = "h{0}".format(header_level) + tag = u"h{0}".format(header_level) self.HandleHtmlOpen(input_line, output_stream, tag, {}, False) else: self._Write("#" * header_level + " ", output_stream) @@ -133,7 +133,7 @@ def HandleHeaderClose( header_level: The header level. """ if self._in_html: - tag = "h{0}".format(header_level) + tag = u"h{0}".format(header_level) self.HandleHtmlClose(input_line, output_stream, tag) else: if self._symmetric_headers: @@ -167,7 +167,7 @@ def HandleCodeBlockOpen(self, input_line, output_stream, specified_language): else: if not specified_language: specified_language = "" - self._Write("```{0}\n".format(specified_language), output_stream) + self._Write(u"```{0}\n".format(specified_language), output_stream) self._in_code_block = True def HandleCodeBlockClose(self, input_line, output_stream): @@ -349,7 +349,7 @@ def HandleSuperscript(self, unused_input_line, output_stream, text): text: The text to output. """ # Markdown currently has no dedicated markup for superscript. - self._Write("{0}".format(text), output_stream) + self._Write(u"{0}".format(text), output_stream) def HandleSubscript(self, unused_input_line, output_stream, text): """Handle the output for subscript text. @@ -360,7 +360,7 @@ def HandleSubscript(self, unused_input_line, output_stream, text): text: The text to output. """ # Markdown currently has no dedicated markup for subscript. - self._Write("{0}".format(text), output_stream) + self._Write(u"{0}".format(text), output_stream) def HandleInlineCode(self, input_line, output_stream, code): """Handle the output for a code block. @@ -391,7 +391,7 @@ def HandleInlineCode(self, input_line, output_stream, code): consecutive_ticks = 0 surrounding_ticks = "`" * (max_consecutive_ticks + 1) - self._Write("{0}{1}{0}".format(surrounding_ticks, code), output_stream) + self._Write(u"{0}{1}{0}".format(surrounding_ticks, code), output_stream) def HandleTableCellBorder(self, input_line, output_stream): """Handle the output for a table cell border. @@ -476,7 +476,7 @@ def HandleTableHeader(self, input_line, output_stream, columns): self.HandleTableCellBorder(input_line, output_stream) # Wiki tables are left-aligned, which takes one character to specify. - self._Write(":{0}".format("-" * (column_width - 1)), output_stream) + self._Write(u":{0}".format("-" * (column_width - 1)), output_stream) self.HandleTableCellBorder(input_line, output_stream) @@ -553,7 +553,7 @@ def HandleLink(self, input_line, output_stream, target, description): # make the link description an inlined image. We do this by setting # the output description to the syntax used to inline an image. if is_image_description: - description = "![]({0})".format(description) + description = u"![]({0})".format(description) elif description: description = self._Escape(description) else: @@ -563,7 +563,7 @@ def HandleLink(self, input_line, output_stream, target, description): # Prefix ! if linking to an image without a text description. prefix = "!" if is_image and is_image_description else "" - output = "{0}[{1}]({2})".format(prefix, description, target) + output = u"{0}[{1}]({2})".format(prefix, description, target) self._Write(output, output_stream) def HandleWiki(self, input_line, output_stream, target, text): @@ -599,7 +599,7 @@ def HandleIssue(self, input_line, output_stream, prefix, issue): input_line, output_stream, migrated_issue_url, - "{0}{1}".format(prefix, migrated_issue)) + u"{0}{1}".format(prefix, migrated_issue)) handled = True instructions = ("In the output, it has been linked to the migrated issue " @@ -626,7 +626,7 @@ def HandleIssue(self, input_line, output_stream, prefix, issue): input_line, output_stream, old_link, - "{0}{1}".format(prefix, issue)) + u"{0}{1}".format(prefix, issue)) handled = True instructions += ("As a placeholder, the text has been modified to " @@ -639,7 +639,7 @@ def HandleIssue(self, input_line, output_stream, prefix, issue): # Couldn't map it to GitHub nor could we link to the old issue. if not handled: - output = "{0}{1} (on Google Code)".format(prefix, issue) + output = u"{0}{1} (on Google Code)".format(prefix, issue) self._Write(output, output_stream) handled = True @@ -649,7 +649,7 @@ def HandleIssue(self, input_line, output_stream, prefix, issue): self._warning_method( input_line, - "Issue {0} was auto-linked. {1}".format(issue, instructions)) + u"Issue {0} was auto-linked. {1}".format(issue, instructions)) def HandleRevision(self, input_line, output_stream, prefix, revision): """Handle the output for an auto-linked issue. @@ -669,13 +669,13 @@ def HandleRevision(self, input_line, output_stream, prefix, revision): input_line, output_stream, old_link, - "{0}{1}".format(prefix, revision)) + u"{0}{1}".format(prefix, revision)) instructions = ("As a placeholder, the text has been modified to " "link to the original Google Code source page:\n\t{0}" .format(old_link)) else: - output = "{0}{1} (on Google Code)".format(prefix, revision) + output = u"{0}{1} (on Google Code)".format(prefix, revision) self._Write(output, output_stream) instructions = ("Additionally, because no project name was specified " @@ -707,12 +707,12 @@ def HandleHtmlOpen( has_end: True if the tag was self-closed. """ core_params = self._SerializeHtmlParams(params) - core = "{0}{1}".format(html_tag, core_params) + core = u"{0}{1}".format(html_tag, core_params) if has_end: - output = "<{0} />".format(core) + output = u"<{0} />".format(core) else: - output = "<{0}>".format(core) + output = u"<{0}>".format(core) self._in_html += 1 self._Write(output, output_stream) @@ -727,7 +727,7 @@ def HandleHtmlClose(self, unused_input_line, output_stream, html_tag): output_stream: Output Markdown file. html_tag: The HTML tag name. """ - self._Write("".format(html_tag), output_stream) + self._Write(u"".format(html_tag), output_stream) self._in_html -= 1 self._has_written_text = False @@ -844,7 +844,7 @@ def _PrintHtmlWarning(self, input_line, kind): input_line, "{0} markup was used within HTML tags. Because GitHub does not " "support this, the tags have been translated to HTML. Please verify " - "that the formatting is correct.".format(kind)) + u"that the formatting is correct.".format(kind)) def _HandleHtmlListOpen( self, @@ -957,10 +957,10 @@ def _HandleFormatClose(self, input_line, output_stream, kind): self.HandleHtmlClose(input_line, output_stream, tag) else: tag = self._HTML_FORMAT_TAGS[kind]["Markdown"] - self._Write("{0}{1}{0}".format(tag, format_buffer), output_stream) + self._Write(u"{0}{1}{0}".format(tag, format_buffer), output_stream) else: - self._warning_method(input_line, "Re-closed '{0}', ignoring.".format(tag)) + self._warning_method(input_line, u"Re-closed '{0}', ignoring.".format(tag)) def _Indent(self, output_stream, indentation_level): """Output indentation. @@ -989,7 +989,7 @@ def _Escape(self, text): before_match = text[:match.start()] after_match = text[match.end():] escaped_match = match.group(0).replace("<", "<").replace(">", ">") - text = "{0}{1}{2}".format(before_match, escaped_match, after_match) + text = u"{0}{1}{2}".format(before_match, escaped_match, after_match) # In Markdown, if a newline is preceeded by two spaces it breaks the line. # For Wiki text, this is not the case, so we strip such endings off. @@ -1012,7 +1012,7 @@ def _SerializeHtmlParams(self, params): quote = "'" else: quote = "\"" - core_params += " {0}={1}{2}{1}".format(name, quote, value) + core_params += u" {0}={1}{2}{1}".format(name, quote, value) return core_params diff --git a/wiki_to_md/wiki2gfm_test.py b/wiki_to_md/wiki2gfm_test.py index cd3594e..845385d 100644 --- a/wiki_to_md/wiki2gfm_test.py +++ b/wiki_to_md/wiki2gfm_test.py @@ -393,7 +393,7 @@ def testHandleSubscriptInHtml(self): def testHandleInlineCode(self): self.formatting_handler.HandleInlineCode(1, self.output, "xyz") - self.assertOutput("` xyz `") + self.assertOutput("`xyz`") self.assertNoWarnings() def testHandleInlineCodeInHtml(self): From 295405377982fadbd536f4ea98ab254f578e6bda Mon Sep 17 00:00:00 2001 From: Nicholas Gorski Date: Mon, 16 Mar 2015 11:06:57 -0700 Subject: [PATCH 17/34] Fix more instances of Unicode format strings. --- wiki_to_md/impl/converter.py | 42 +++++++++++++-------------- wiki_to_md/impl/formatting_handler.py | 22 +++++++------- wiki_to_md/impl/pragma_handler.py | 6 ++-- wiki_to_md/wiki2gfm.py | 2 +- 4 files changed, 36 insertions(+), 36 deletions(-) diff --git a/wiki_to_md/impl/converter.py b/wiki_to_md/impl/converter.py index bcbda64..f1128d5 100644 --- a/wiki_to_md/impl/converter.py +++ b/wiki_to_md/impl/converter.py @@ -127,8 +127,8 @@ def Convert(self, input_stream, output_stream): if remaining_lines != 0: self._warning_method( input_line, - "Processing completed, but not all lines were processed. " - u"Remaining lines: {0}.".format(remaining_lines)) + u"Processing completed, but not all lines were processed. " + "Remaining lines: {0}.".format(remaining_lines)) def _ExtractPragmas(self, input_line, input_lines, output_stream): """Extracts pragmas from a given input. @@ -320,7 +320,7 @@ def _ProcessLine( if line[indent_pos + 1] != " ": self._warning_method( input_line, - "Missing space after list symbol: {0}, " + u"Missing space after list symbol: {0}, " "'{1}' was removed instead." .format(line[indent_pos], line[indent_pos + 1])) line = line[indent_pos + 2:] @@ -935,8 +935,8 @@ def _HandlePlugin(self, input_line, match, output_stream): else: self._warning_method( input_line, - "Unknown plugin was given, outputting " - u"as plain text:\n\t{0}".format(match)) + u"Unknown plugin was given, outputting " + "as plain text:\n\t{0}".format(match)) # Wiki syntax put this class of error on its own line. self._formatting_handler.HandleEscapedText( input_line, @@ -974,7 +974,7 @@ def _HandlePluginHtml( else: self._warning_method( input_line, - "The following parameter was given for the '{0}' tag, " + u"The following parameter was given for the '{0}' tag, " "but will not be present in the outputted HTML:\n\t'{1}': '{2}'" .format(plugin_id, name, value)) @@ -1012,7 +1012,7 @@ def _HandlePluginGPlus( else: self._warning_method( input_line, - "The following parameter was given for the '{0}' tag, " + u"The following parameter was given for the '{0}' tag, " "but will not be present in the outputted HTML:\n\t'{1}': '{2}'" .format(plugin_id, name, value)) @@ -1038,7 +1038,7 @@ def _HandlePluginWikiComment( for name, value in params.items(): self._warning_method( input_line, - "The following parameter was given for the '{0}' tag, " + u"The following parameter was given for the '{0}' tag, " "but will not be present in the outputted HTML:\n\t'{1}': '{2}'" .format(plugin_id, name, value)) @@ -1054,7 +1054,7 @@ def _HandlePluginWikiGadget(self, input_line, match, output_stream): """ self._warning_method( input_line, - "A wiki gadget was used, but this must be manually converted to a " + u"A wiki gadget was used, but this must be manually converted to a " "GFM-supported method, if possible. Outputting as plain text:\n\t{0}" .format(match)) self._formatting_handler.HandleEscapedText( @@ -1083,7 +1083,7 @@ def _HandlePluginWikiVideo( else: self._warning_method( input_line, - "The following parameter was given for the '{0}' tag, " + u"The following parameter was given for the '{0}' tag, " "but will not be present in the outputted HTML:\n\t'{1}': '{2}'" .format(plugin_id, name, value)) @@ -1100,7 +1100,7 @@ def _HandlePluginWikiVideo( "video id within parameter \"url\".") self._warning_method( input_line, - "Video plugin has invalid video ID, outputting error:\n\t{0}" + u"Video plugin has invalid video ID, outputting error:\n\t{0}" .format(output)) # Wiki syntax put this class of error on its own line. self._formatting_handler.HandleEscapedText( @@ -1118,7 +1118,7 @@ def _HandlePluginWikiVideo( output = "wiki:video: missing mandatory parameter \"url\"." self._warning_method( input_line, - "Video plugin is missing 'url' parameter, outputting error:\n\t{0}" + u"Video plugin is missing 'url' parameter, outputting error:\n\t{0}" .format(output)) # Wiki syntax put this class of error on its own line. self._formatting_handler.HandleEscapedText( @@ -1136,7 +1136,7 @@ def _HandlePluginWikiToc(self, input_line, match, output_stream): """ self._warning_method( input_line, - "A table of contents plugin was used for this wiki:\n" + u"A table of contents plugin was used for this wiki:\n" "\t{0}\n" "The Gollum wiki system supports table of content generation.\n" "See https://github.com/gollum/gollum/wiki for more information.\n" @@ -1185,8 +1185,8 @@ def _HandlePluginEnd(self, input_line, match, output_stream): else: self._warning_method( input_line, - "Unknown but matching plugin end was given, outputting " - u"as plain text:\n\t{0}".format(match)) + u"Unknown but matching plugin end was given, outputting " + "as plain text:\n\t{0}".format(match)) # Wiki syntax put this class of error on its own line. self._formatting_handler.HandleEscapedText( input_line, @@ -1195,8 +1195,8 @@ def _HandlePluginEnd(self, input_line, match, output_stream): else: self._warning_method( input_line, - "Unknown/unmatched plugin end was given, outputting " - u"as plain text with errors:\n\t{0}".format(match)) + u"Unknown/unmatched plugin end was given, outputting " + "as plain text with errors:\n\t{0}".format(match)) # Wiki syntax put this class of error on its own line, # with a prefix error message, and did not display the tag namespace. tag_without_ns = plugin_id.split(":", 1)[-1] @@ -1245,12 +1245,12 @@ def _HandleVariable(self, input_line, match, output_stream): elif not output and match == "project": if self._project: output = self._project - instructions = ("It has been replaced with static text containing the " - u"name of the project:\n\t{0}".format(self._project)) + instructions = (u"It has been replaced with static text containing the " + "name of the project:\n\t{0}".format(self._project)) else: output = "(TODO: Replace with project name.)" instructions = ("Because no project name was specified, the text has " - u"been replaced with:\n\t{0}".format(output)) + "been replaced with:\n\t{0}".format(output)) # Not defined anywhere, just treat as regular text. if not output: @@ -1264,7 +1264,7 @@ def _HandleVariable(self, input_line, match, output_stream): if instructions: self._warning_method( input_line, - "A variable substitution was performed with %%{0}%%. {1}" + u"A variable substitution was performed with %%{0}%%. {1}" .format(match, instructions)) def _HandleTag(self, input_line, tag, output_stream): diff --git a/wiki_to_md/impl/formatting_handler.py b/wiki_to_md/impl/formatting_handler.py index 04c4763..664fc26 100644 --- a/wiki_to_md/impl/formatting_handler.py +++ b/wiki_to_md/impl/formatting_handler.py @@ -30,7 +30,7 @@ class FormattingHandler(object): # Template for linking to a video. _VIDEO_TEMPLATE = ( - "") @@ -602,7 +602,7 @@ def HandleIssue(self, input_line, output_stream, prefix, issue): u"{0}{1}".format(prefix, migrated_issue)) handled = True - instructions = ("In the output, it has been linked to the migrated issue " + instructions = (u"In the output, it has been linked to the migrated issue " "on GitHub: {0}. Please verify this issue on GitHub " "corresponds to the original issue on Google Code. " .format(migrated_issue)) @@ -620,7 +620,7 @@ def HandleIssue(self, input_line, output_stream, prefix, issue): # If we couldn't handle it in the map, try linking to the old issue. if not handled and self._project: - old_link = ("https://code.google.com/p/{0}/issues/detail?id={1}" + old_link = (u"https://code.google.com/p/{0}/issues/detail?id={1}" .format(self._project, issue)) self.HandleLink( input_line, @@ -629,7 +629,7 @@ def HandleIssue(self, input_line, output_stream, prefix, issue): u"{0}{1}".format(prefix, issue)) handled = True - instructions += ("As a placeholder, the text has been modified to " + instructions += (u"As a placeholder, the text has been modified to " "link to the original Google Code issue page:\n\t{0}" .format(old_link)) elif not handled: @@ -643,7 +643,7 @@ def HandleIssue(self, input_line, output_stream, prefix, issue): self._Write(output, output_stream) handled = True - instructions += ("The auto-link has been removed and the text has been " + instructions += (u"The auto-link has been removed and the text has been " "modified from '{0}{1}' to '{2}'." .format(prefix, issue, output)) @@ -663,7 +663,7 @@ def HandleRevision(self, input_line, output_stream, prefix, revision): # Google Code only auto-linked revision numbers, not hashes, so # revision auto-linking cannot be done for the conversion. if self._project: - old_link = ("https://code.google.com/p/{0}/source/detail?r={1}" + old_link = (u"https://code.google.com/p/{0}/source/detail?r={1}" .format(self._project, revision)) self.HandleLink( input_line, @@ -671,14 +671,14 @@ def HandleRevision(self, input_line, output_stream, prefix, revision): old_link, u"{0}{1}".format(prefix, revision)) - instructions = ("As a placeholder, the text has been modified to " + instructions = (u"As a placeholder, the text has been modified to " "link to the original Google Code source page:\n\t{0}" .format(old_link)) else: output = u"{0}{1} (on Google Code)".format(prefix, revision) self._Write(output, output_stream) - instructions = ("Additionally, because no project name was specified " + instructions = (u"Additionally, because no project name was specified " "the revision could not be linked to the original " "Google Code source page. The auto-link has been removed " "and the text has been modified from '{0}{1}' to '{2}'." @@ -686,7 +686,7 @@ def HandleRevision(self, input_line, output_stream, prefix, revision): self._warning_method( input_line, - "Revision {0} was auto-linked. SVN revision numbers are not sensible " + u"Revision {0} was auto-linked. SVN revision numbers are not sensible " "in Git; consider updating this link or removing it altogether. {1}" .format(revision, instructions)) @@ -842,9 +842,9 @@ def _PrintHtmlWarning(self, input_line, kind): """ self._warning_method( input_line, - "{0} markup was used within HTML tags. Because GitHub does not " + u"{0} markup was used within HTML tags. Because GitHub does not " "support this, the tags have been translated to HTML. Please verify " - u"that the formatting is correct.".format(kind)) + "that the formatting is correct.".format(kind)) def _HandleHtmlListOpen( self, diff --git a/wiki_to_md/impl/pragma_handler.py b/wiki_to_md/impl/pragma_handler.py index 006bd2d..f2f12c6 100644 --- a/wiki_to_md/impl/pragma_handler.py +++ b/wiki_to_md/impl/pragma_handler.py @@ -44,14 +44,14 @@ def HandlePragma(self, if pragma_type == "summary": self._warning_method( input_line, - "A summary pragma was used for this wiki:\n" + u"A summary pragma was used for this wiki:\n" "\t{0}\n" "Consider moving it to an introductory paragraph." .format(pragma_value)) elif pragma_type == "sidebar": self._warning_method( input_line, - "A sidebar pragma was used for this wiki:\n" + u"A sidebar pragma was used for this wiki:\n" "\t{0}\n" "The Gollum wiki system supports sidebars, and by converting " "{0}.wiki to _Sidebar.md it can be used as a sidebar.\n" @@ -60,7 +60,7 @@ def HandlePragma(self, else: self._warning_method( input_line, - "The following pragma has been ignored:\n" + u"The following pragma has been ignored:\n" "\t#{0} {1}\n" "Consider expressing the same information in a different manner." .format(pragma_type, pragma_value)) diff --git a/wiki_to_md/wiki2gfm.py b/wiki_to_md/wiki2gfm.py index 60ab35a..8ee84ec 100755 --- a/wiki_to_md/wiki2gfm.py +++ b/wiki_to_md/wiki2gfm.py @@ -45,7 +45,7 @@ def PrintWarning(input_line, message): input_line: The line number this warning occurred on. message: The warning message. """ - print "Warning (line {0} of input file):\n{1}\n".format(input_line, message) + print u"Warning (line {0} of input file):\n{1}\n".format(input_line, message) def main(args): From 1f907dcefcaa468d5f5d89d3e305a49d19b6cfe7 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 24 Apr 2015 10:20:20 -0700 Subject: [PATCH 18/34] Change convert-repo.sh to be executable. --- wiki_to_md/convert-repo.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 wiki_to_md/convert-repo.sh diff --git a/wiki_to_md/convert-repo.sh b/wiki_to_md/convert-repo.sh old mode 100644 new mode 100755 From 1f416a55dc570f44bb3058046d1549f5b513842e Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 24 Apr 2015 10:50:23 -0700 Subject: [PATCH 19/34] Add file attachments to issue export. --- googlecode-issues-exporter/issues.py | 32 +++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index a49ed4c..a3fbef6 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -253,7 +253,37 @@ def GetContent(self): Returns: The issue comment """ - return self._comment["content"] + body = self._comment["content"] + + # Add references to attachments as appropriate. + attachmentLines = [] + for attachment in self._comment["attachments"] if "attachments" in self._comment else []: + if "isDeleted" in attachment: + line = " * *Attachment: %s (deleted)*" % (attachment["fileName"]) + attachmentLines.append(line) + continue + + link = "https://storage.googleapis.com/google-code-attachments/%s/issue-%d/comment-%d/%s" % ( + self.GetIssue().GetProjectName(), self.GetIssue().GetId(), + self.GetId(), attachment["fileName"]) + + def has_extension(extension): + return attachment["fileName"].lower().endswith(extension) + + is_image_attachment = False + for extension in [".png", ".jpg", ".jpeg", ".bmp", ".tif", ".gif"]: + is_image_attachment |= has_extension(".png") + + if is_image_attachment: + line = " * *Attachment: %s
![%s](%s)*" % ( + attachment["fileName"], attachment["fileName"], link) + else: + line = " * *Attachment: [%s](%s)*" % (attachment["fileName"], link) + attachmentLines.append(line) + + if len(attachmentLines) > 0: + body += "\n
\n" + "\n".join(attachmentLines) + return body def GetCreatedOn(self): """Get the creation date from a Google Code comment. From 5c74f246f346d54b76e4346767c0947b9f40af83 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 24 Apr 2015 13:48:12 -0700 Subject: [PATCH 20/34] Fix bug with whitespace issue titles --- googlecode-issues-exporter/github_issue_converter.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/googlecode-issues-exporter/github_issue_converter.py b/googlecode-issues-exporter/github_issue_converter.py index 637b085..fa49e3e 100644 --- a/googlecode-issues-exporter/github_issue_converter.py +++ b/googlecode-issues-exporter/github_issue_converter.py @@ -327,6 +327,10 @@ def CreateIssue(self, googlecode_issue): issues.ServiceError: An error occurred creating the issue. """ issue_title = googlecode_issue.GetTitle() + # It is not possible to create a Google Code issue without a title, but you + # can edit an issue to remove its title afterwards. + if issue_title.isspace(): + issue_title = "" issue = { "title": issue_title, "body": googlecode_issue.GetDescription(), @@ -339,8 +343,8 @@ def CreateIssue(self, googlecode_issue): if not _CheckSuccessful(response): # Newline character at the beginning of the line to allows for in-place # updating of the counts of the issues and comments. - raise issues.ServiceError("\nFailed to create issue: %s.\n%s" % ( - issue_title, content)) + raise issues.ServiceError("\nFailed to create issue %d: %s.\n%s" % ( + googlecode_issue.GetId(), issue_title, content)) return self._GetIssueNumber(content) From 2098b3e3ec1078cbcfb062f7f5af6d0c8194cee1 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 10:31:19 -0700 Subject: [PATCH 21/34] Fix unit test, refactor recording of previously exported issues. --- .../github_issue_converter.py | 2 +- .../github_issue_converter_test.py | 117 ++++++++++-------- googlecode-issues-exporter/issues.py | 28 +++-- 3 files changed, 80 insertions(+), 67 deletions(-) diff --git a/googlecode-issues-exporter/github_issue_converter.py b/googlecode-issues-exporter/github_issue_converter.py index fa49e3e..3ed173a 100644 --- a/googlecode-issues-exporter/github_issue_converter.py +++ b/googlecode-issues-exporter/github_issue_converter.py @@ -400,7 +400,7 @@ def _GetIssueNumber(self, content): Returns: The GitHub issue number. """ - assert "number" in content, "nope %s" % content + assert "number" in content, "Error getting GH issue no from: %s" % content return content["number"] diff --git a/googlecode-issues-exporter/github_issue_converter_test.py b/googlecode-issues-exporter/github_issue_converter_test.py index 365ca13..df3bd5d 100644 --- a/googlecode-issues-exporter/github_issue_converter_test.py +++ b/googlecode-issues-exporter/github_issue_converter_test.py @@ -166,7 +166,7 @@ def __init__(self): self.last_method = None self.last_body = None - def request(self, url, method, body=None): + def request(self, url, method, headers=None, body=None): """Makes a fake HTTP request. Args: @@ -190,6 +190,7 @@ def setUp(self): self.http_mock = Http2Mock() self.github_service = github_issue_converter.GitHubService( GITHUB_USERNAME, GITHUB_REPO, GITHUB_TOKEN, + rate_limit=False, http_instance=self.http_mock) def testSuccessfulRequestSuccess(self): @@ -303,6 +304,7 @@ def setUp(self): self.http_mock = Http2Mock() self.github_service = github_issue_converter.GitHubService( GITHUB_USERNAME, GITHUB_REPO, GITHUB_TOKEN, + rate_limit=False, http_instance=self.http_mock) self.github_issue_service = github_issue_converter.IssueService( self.github_service, comment_delay=0) @@ -378,14 +380,53 @@ def setUp(self): NO_ISSUE_DATA, GITHUB_REPO, USER_MAP) self.issue_exporter.Init() + self.TEST_ISSUE_DATA = [ + { + "id": "1", + "title": "Title1", + "state": "open", + "comments": { + "items": [COMMENT_ONE, COMMENT_TWO, COMMENT_THREE], + }, + "labels": ["Type-Issue", "Priority-High"], + "owner": {"kind": "projecthosting#issuePerson", + "name": "User1" + }, + }, + { + "id": "2", + "title": "Title2", + "state": "closed", + "owner": {"kind": "projecthosting#issuePerson", + "name": "User2" + }, + "labels": [], + "comments": { + "items": [COMMENT_ONE], + }, + }, + { + "id": "3", + "title": "Title3", + "state": "closed", + "comments": { + "items": [COMMENT_ONE, COMMENT_TWO], + }, + "labels": ["Type-Defect"], + "owner": {"kind": "projecthosting#issuePerson", + "name": "User3" + } + }] + + def testGetAllPreviousIssues(self): self.assertEqual(0, len(self.issue_exporter._previously_created_issues)) - content = [{"title": "issue_title"}] + content = [{"id": 1, "title": "issue_title"}] self.github_service.AddResponse(content=content) self.issue_exporter._GetAllPreviousIssues() self.assertEqual(1, len(self.issue_exporter._previously_created_issues)) - self.assertTrue("issue_title" in - self.issue_exporter._previously_created_issues) + self.assertTrue(1 in self.issue_exporter._previously_created_issues) + self.assertEqual("issue_title", self.issue_exporter._previously_created_issues[1]) def testCreateIssue(self): content = {"number": 1234} @@ -418,52 +459,16 @@ def testCreateCommentsFailure(self): self.issue_exporter._CreateComments(COMMENTS_DATA, 1234, SINGLE_ISSUE) def testStart(self): - self.issue_exporter._issue_json_data = [ - { - "id": "1", - "title": "Title1", - "state": "open", - "comments": { - "items": [COMMENT_ONE, COMMENT_TWO, COMMENT_THREE], - }, - "labels": ["Type-Issue", "Priority-High"], - "owner": {"kind": "projecthosting#issuePerson", - "name": "User1" - }, - }, - { - "id": "2", - "title": "Title2", - "state": "closed", - "owner": {"kind": "projecthosting#issuePerson", - "name": "User2" - }, - "labels": [], - "comments": { - "items": [COMMENT_ONE], - }, - }, - { - "id": "3", - "title": "Title3", - "state": "closed", - "comments": { - "items": [COMMENT_ONE, COMMENT_TWO], - }, - "labels": ["Type-Defect"], - "owner": {"kind": "projecthosting#issuePerson", - "name": "User3" - } - }] + self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA - self.github_service.AddResponse(content={"number": 1234}) - self.github_service.AddResponse(content={"number": 2345}) - self.github_service.AddResponse(content={"number": 3456}) - self.github_service.AddResponse(content={"number": 5678}) - self.github_service.AddResponse(content={"number": 6789}) - self.github_service.AddResponse(content={"number": 7890}) - self.github_service.AddResponse(content={"number": 8901}) - self.github_service.AddResponse(content={"number": 9012}) + # Note: Some responses are from CreateIssues, others are from CreateComment. + self.github_service.AddResponse(content={"number": 1}) + self.github_service.AddResponse(content={"number": 10}) + self.github_service.AddResponse(content={"number": 11}) + self.github_service.AddResponse(content={"number": 12}) + self.github_service.AddResponse(content={"number": 2}) + self.github_service.AddResponse(content={"number": 30}) + self.github_service.AddResponse(content={"number": 31}) self.issue_exporter.Start() @@ -476,11 +481,17 @@ def testStart(self): self.assertEqual(1, self.issue_exporter._comment_total) def testStartSkipAlreadyCreatedIssues(self): - self.issue_exporter._previously_created_issues.add("Title1") - self.issue_exporter._issue_json_data = [{"title": "Title1"}] + self.issue_exporter._previously_created_issues["1"] = "Title1" + self.issue_exporter._previously_created_issues["2"] = "Title2" + self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA + + self.github_service.AddResponse(content={"number": 3}) + self.github_service.AddResponse(content={"number": 4}) + self.issue_exporter.Start() - self.assertEqual(1, self.issue_exporter._issue_total) - self.assertEqual(0, self.issue_exporter._issue_number) + self.assertEqual(2, self.issue_exporter._skipped_issues) + self.assertEqual(3, self.issue_exporter._issue_total) + self.assertEqual(3, self.issue_exporter._issue_number) if __name__ == "__main__": unittest.main(buffer=True) diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index a3fbef6..b220653 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -500,29 +500,31 @@ def __init__(self, issue_service, user_service, issue_json_data, self._project_name = project_name self._user_map = user_map - self._previously_created_issues = set() + # Mapping from issue ID to the issue's title. This is used to verify + # consistency with an previous attempts at exporting issues. + self._previously_created_issues = {} self._issue_total = 0 self._issue_number = 0 self._comment_number = 0 self._comment_total = 0 + self._skipped_issues = 0 def Init(self): """Initialize the needed variables.""" self._GetAllPreviousIssues() def _GetAllPreviousIssues(self): - """Gets all previously uploaded issues. - - Creates a hash of the issue titles, they will be unique as the Google Code - issue number is in each title. - """ + """Gets all previously uploaded issues.""" print "Getting any previously added issues..." open_issues = self._issue_service.GetIssues("open") closed_issues = self._issue_service.GetIssues("closed") issues = open_issues + closed_issues for issue in issues: - self._previously_created_issues.add(issue["title"]) + print "%s" % issue + issue["id"] not in self._previously_created_issues or die( + "GitHub returned multiple issues with the same ID?") + self._previously_created_issues[issue["id"]] = issue["title"] def _UpdateProgressBar(self): """Update issue count 'feed'. @@ -585,17 +587,17 @@ def Start(self): self._issue_total = len(self._issue_json_data) self._issue_number = 0 - skipped_issues = 0 + self._skipped_issues = 0 for issue in self._issue_json_data: googlecode_issue = GoogleCodeIssue( issue, self._project_name, self._user_map) - issue_title = googlecode_issue.GetTitle() self._issue_number += 1 self._UpdateProgressBar() - if issue_title in self._previously_created_issues: - skipped_issues += 1 + issue_id = googlecode_issue.GetId() + if issue_id in self._previously_created_issues: + self._skipped_issues += 1 continue issue_number = self._CreateIssue(googlecode_issue) @@ -608,6 +610,6 @@ def Start(self): if not googlecode_issue.IsOpen(): self._issue_service.CloseIssue(issue_number) - if skipped_issues > 0: + if self._skipped_issues > 0: print ("\nSkipped %d/%d issue previously uploaded." % - (skipped_issues, self._issue_total)) + (self._skipped_issues, self._issue_total)) From ad496f980bdbe005583bc1e72e8886d2202c48f6 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 10:46:00 -0700 Subject: [PATCH 22/34] Raise error if partial results don't match. --- .../github_issue_converter_test.py | 11 +++++++++- googlecode-issues-exporter/issues.py | 22 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/googlecode-issues-exporter/github_issue_converter_test.py b/googlecode-issues-exporter/github_issue_converter_test.py index df3bd5d..ef082d5 100644 --- a/googlecode-issues-exporter/github_issue_converter_test.py +++ b/googlecode-issues-exporter/github_issue_converter_test.py @@ -480,7 +480,7 @@ def testStart(self): self.assertEqual(1, self.issue_exporter._comment_number) self.assertEqual(1, self.issue_exporter._comment_total) - def testStartSkipAlreadyCreatedIssues(self): + def testStart_SkipAlreadyCreatedIssues(self): self.issue_exporter._previously_created_issues["1"] = "Title1" self.issue_exporter._previously_created_issues["2"] = "Title2" self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA @@ -493,5 +493,14 @@ def testStartSkipAlreadyCreatedIssues(self): self.assertEqual(3, self.issue_exporter._issue_total) self.assertEqual(3, self.issue_exporter._issue_number) + def testStart_GetErrorIfGoogleCodeAndGitHubDoNotMatch(self): + self.issue_exporter._previously_created_issues["1"] = "Title1" + self.issue_exporter._previously_created_issues["2"] = "" + self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA + + with self.assertRaises(RuntimeError): + self.issue_exporter.Start() + + if __name__ == "__main__": unittest.main(buffer=True) diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index b220653..af9c17f 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -585,6 +585,27 @@ def Start(self): print ("Existing issues detected for the repo. Likely due to" " the script being previously aborted or killed.") + # Verify the last GitHub issue matches the Google Code issue. Also + # that it has all of the expected comments. + last_gh_issue_id = -1 + for id in self._previously_created_issues: + if id > last_gh_issue_id: + last_gh_issue_id = id + last_gh_issue_title = self._previously_created_issues[last_gh_issue_id] + + last_gc_issue = None + for issue in self._issue_json_data: + if issue["id"] == last_gh_issue_id and ( + issue["title"] == last_gh_issue_title): + last_gc_issue = issue + break + + if last_gc_issue is None: + raise RuntimeError("Unable to find Google Code issue #%s '%s'." % ( + last_gh_issue_id, last_gh_issue_title)) + + print "Found last issues, they match." + self._issue_total = len(self._issue_json_data) self._issue_number = 0 self._skipped_issues = 0 @@ -610,6 +631,7 @@ def Start(self): if not googlecode_issue.IsOpen(): self._issue_service.CloseIssue(issue_number) + # TODO(chrsmith): Issue a warning if/when the issue ID get out of sync. if self._skipped_issues > 0: print ("\nSkipped %d/%d issue previously uploaded." % (self._skipped_issues, self._issue_total)) From 392d7eb74552c94a23e6e8558e19f27c8e1673eb Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 11:27:24 -0700 Subject: [PATCH 23/34] Check if any comments missed. --- .../github_issue_converter.py | 2 +- .../github_issue_converter_test.py | 61 ++++++++++++++++--- googlecode-issues-exporter/issues.py | 44 ++++++++++--- 3 files changed, 88 insertions(+), 19 deletions(-) diff --git a/googlecode-issues-exporter/github_issue_converter.py b/googlecode-issues-exporter/github_issue_converter.py index 3ed173a..4a16091 100644 --- a/googlecode-issues-exporter/github_issue_converter.py +++ b/googlecode-issues-exporter/github_issue_converter.py @@ -400,7 +400,7 @@ def _GetIssueNumber(self, content): Returns: The GitHub issue number. """ - assert "number" in content, "Error getting GH issue no from: %s" % content + assert "number" in content, "Getting issue number from: %s" % content return content["number"] diff --git a/googlecode-issues-exporter/github_issue_converter_test.py b/googlecode-issues-exporter/github_issue_converter_test.py index ef082d5..e511f5b 100644 --- a/googlecode-issues-exporter/github_issue_converter_test.py +++ b/googlecode-issues-exporter/github_issue_converter_test.py @@ -421,12 +421,15 @@ def setUp(self): def testGetAllPreviousIssues(self): self.assertEqual(0, len(self.issue_exporter._previously_created_issues)) - content = [{"id": 1, "title": "issue_title"}] + content = [{"id": 1, "title": "issue_title", "comments": 2}] self.github_service.AddResponse(content=content) self.issue_exporter._GetAllPreviousIssues() self.assertEqual(1, len(self.issue_exporter._previously_created_issues)) self.assertTrue(1 in self.issue_exporter._previously_created_issues) - self.assertEqual("issue_title", self.issue_exporter._previously_created_issues[1]) + self.assertEqual("issue_title", + self.issue_exporter._previously_created_issues[1]["title"]) + self.assertEqual(2, + self.issue_exporter._previously_created_issues[1]["comment_count"]) def testCreateIssue(self): content = {"number": 1234} @@ -465,10 +468,10 @@ def testStart(self): self.github_service.AddResponse(content={"number": 1}) self.github_service.AddResponse(content={"number": 10}) self.github_service.AddResponse(content={"number": 11}) - self.github_service.AddResponse(content={"number": 12}) self.github_service.AddResponse(content={"number": 2}) + self.github_service.AddResponse(content={"number": 20}) + self.github_service.AddResponse(content={"number": 3}) self.github_service.AddResponse(content={"number": 30}) - self.github_service.AddResponse(content={"number": 31}) self.issue_exporter.Start() @@ -481,23 +484,63 @@ def testStart(self): self.assertEqual(1, self.issue_exporter._comment_total) def testStart_SkipAlreadyCreatedIssues(self): - self.issue_exporter._previously_created_issues["1"] = "Title1" - self.issue_exporter._previously_created_issues["2"] = "Title2" + self.issue_exporter._previously_created_issues["1"] = { + "title": "Title1", + "comment_count": 3 + } + self.issue_exporter._previously_created_issues["2"] = { + "title": "Title2", + "comment_count": 1 + } self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA self.github_service.AddResponse(content={"number": 3}) - self.github_service.AddResponse(content={"number": 4}) self.issue_exporter.Start() self.assertEqual(2, self.issue_exporter._skipped_issues) self.assertEqual(3, self.issue_exporter._issue_total) self.assertEqual(3, self.issue_exporter._issue_number) + def testStart_ReAddMissedComments(self): + self.issue_exporter._previously_created_issues["1"] = { + "title": "Title1", + "comment_count": 1 # Missing 2 comments. + } + self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA + + # First requests to re-add comments, then create issues. + self.github_service.AddResponse(content={"number": 11}) + + self.github_service.AddResponse(content={"number": 2}) + self.github_service.AddResponse(content={"number": 20}) + self.github_service.AddResponse(content={"number": 3}) + self.github_service.AddResponse(content={"number": 30}) + + self.issue_exporter.Start() + self.assertEqual(1, self.issue_exporter._skipped_issues) + self.assertEqual(3, self.issue_exporter._issue_total) + self.assertEqual(3, self.issue_exporter._issue_number) + + def testStart_GetErrorIfGoogleCodeAndGitHubDoNotMatch(self): - self.issue_exporter._previously_created_issues["1"] = "Title1" - self.issue_exporter._previously_created_issues["2"] = "" + self.issue_exporter._previously_created_issues["1"] = { + "title": "Title1"} + self.issue_exporter._previously_created_issues["2"] = { + "title": "< not issue #2's title >"} + self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA + + with self.assertRaises(RuntimeError): + self.issue_exporter.Start() + + def testStart_GetErrorIfCreatedGitHubIDDoesNotMatch(self): self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA + # Note: Some responses are from CreateIssues, others are from CreateComment. + self.github_service.AddResponse(content={"number": 1}) + self.github_service.AddResponse(content={"number": 10}) + self.github_service.AddResponse(content={"number": 11}) + self.github_service.AddResponse(content={"number": 3}) # Expects next issue ID 2. + with self.assertRaises(RuntimeError): self.issue_exporter.Start() diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index af9c17f..364897e 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -500,7 +500,7 @@ def __init__(self, issue_service, user_service, issue_json_data, self._project_name = project_name self._user_map = user_map - # Mapping from issue ID to the issue's title. This is used to verify + # Mapping from issue ID to the issue's metadata. This is used to verify # consistency with an previous attempts at exporting issues. self._previously_created_issues = {} @@ -524,7 +524,10 @@ def _GetAllPreviousIssues(self): print "%s" % issue issue["id"] not in self._previously_created_issues or die( "GitHub returned multiple issues with the same ID?") - self._previously_created_issues[issue["id"]] = issue["title"] + self._previously_created_issues[issue["id"]] = { + "title": issue["title"], + "comment_count": issue["comments"], + } def _UpdateProgressBar(self): """Update issue count 'feed'. @@ -581,6 +584,7 @@ def Start(self): This will traverse the issues and attempt to create each issue and its comments. """ + # TODO(chris): Break into another function. if len(self._previously_created_issues): print ("Existing issues detected for the repo. Likely due to" " the script being previously aborted or killed.") @@ -591,20 +595,39 @@ def Start(self): for id in self._previously_created_issues: if id > last_gh_issue_id: last_gh_issue_id = id - last_gh_issue_title = self._previously_created_issues[last_gh_issue_id] + last_gh_issue = self._previously_created_issues[last_gh_issue_id] last_gc_issue = None for issue in self._issue_json_data: if issue["id"] == last_gh_issue_id and ( - issue["title"] == last_gh_issue_title): - last_gc_issue = issue + issue["title"] == last_gh_issue["title"]): + last_gc_issue = GoogleCodeIssue( + issue, self._project_name, self._user_map) break if last_gc_issue is None: - raise RuntimeError("Unable to find Google Code issue #%s '%s'." % ( - last_gh_issue_id, last_gh_issue_title)) - - print "Found last issues, they match." + message = "Were GitHub issues added since the last import started?" + raise RuntimeError("Unable to find Google Code issue #%s '%s'.\n%s" % ( + last_gh_issue_id, last_gh_issue["title"], message)) + + print "Last GitHub issue (%s: %s) matches. Checking comments..." % ( + last_gh_issue_id, last_gh_issue["title"]) + + # Verify the last GitHub issue contains all the comments that the + # Google Code issue does. (That they weren't truncated.) + num_gc_issue_comments = len(last_gc_issue.GetComments()) + print "last_gh_issue %s\nlast_gc_issue %s %s %s" % (last_gh_issue, last_gc_issue.GetComments(), last_gc_issue.GetId(), last_gc_issue.GetTitle()) + if last_gh_issue["comment_count"] != num_gc_issue_comments: + print "GitHub issue has fewer comments than Google Code's. Fixing..." + for idx in range(last_gh_issue["comment_count"], num_gc_issue_comments): + comment = last_gc_issue.GetComments()[idx] + googlecode_comment = GoogleCodeComment(last_gc_issue, comment) + self._issue_service.CreateComment( + # issue_number == source_issue_id + int(last_gc_issue.GetId()), int(last_gc_issue.GetId()), + googlecode_comment, self._project_name) + + print "Done! Resuming normal issue execution." self._issue_total = len(self._issue_json_data) self._issue_number = 0 @@ -624,6 +647,9 @@ def Start(self): issue_number = self._CreateIssue(googlecode_issue) if issue_number < 0: continue + if int(issue_number) != int(googlecode_issue.GetId()): + raise RuntimeError("Google Code and GitHub issue nos mismatch at #%s (got %s)" % ( + googlecode_issue.GetId(), issue_number)) comments = googlecode_issue.GetComments() self._CreateComments(comments, issue_number, googlecode_issue) From 5bdbcdea29b4e4f7ea227ebfd16352497c75cef2 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 11:40:43 -0700 Subject: [PATCH 24/34] refactoring --- .../github_issue_converter_test.py | 1 + googlecode-issues-exporter/issues.py | 104 ++++++++++-------- 2 files changed, 61 insertions(+), 44 deletions(-) diff --git a/googlecode-issues-exporter/github_issue_converter_test.py b/googlecode-issues-exporter/github_issue_converter_test.py index e511f5b..643d4b5 100644 --- a/googlecode-issues-exporter/github_issue_converter_test.py +++ b/googlecode-issues-exporter/github_issue_converter_test.py @@ -497,6 +497,7 @@ def testStart_SkipAlreadyCreatedIssues(self): self.github_service.AddResponse(content={"number": 3}) self.issue_exporter.Start() + self.assertEqual(2, self.issue_exporter._skipped_issues) self.assertEqual(3, self.issue_exporter._issue_total) self.assertEqual(3, self.issue_exporter._issue_number) diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index 364897e..67a264b 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -584,50 +584,10 @@ def Start(self): This will traverse the issues and attempt to create each issue and its comments. """ - # TODO(chris): Break into another function. - if len(self._previously_created_issues): - print ("Existing issues detected for the repo. Likely due to" - " the script being previously aborted or killed.") - - # Verify the last GitHub issue matches the Google Code issue. Also - # that it has all of the expected comments. - last_gh_issue_id = -1 - for id in self._previously_created_issues: - if id > last_gh_issue_id: - last_gh_issue_id = id - last_gh_issue = self._previously_created_issues[last_gh_issue_id] - - last_gc_issue = None - for issue in self._issue_json_data: - if issue["id"] == last_gh_issue_id and ( - issue["title"] == last_gh_issue["title"]): - last_gc_issue = GoogleCodeIssue( - issue, self._project_name, self._user_map) - break - - if last_gc_issue is None: - message = "Were GitHub issues added since the last import started?" - raise RuntimeError("Unable to find Google Code issue #%s '%s'.\n%s" % ( - last_gh_issue_id, last_gh_issue["title"], message)) - - print "Last GitHub issue (%s: %s) matches. Checking comments..." % ( - last_gh_issue_id, last_gh_issue["title"]) - - # Verify the last GitHub issue contains all the comments that the - # Google Code issue does. (That they weren't truncated.) - num_gc_issue_comments = len(last_gc_issue.GetComments()) - print "last_gh_issue %s\nlast_gc_issue %s %s %s" % (last_gh_issue, last_gc_issue.GetComments(), last_gc_issue.GetId(), last_gc_issue.GetTitle()) - if last_gh_issue["comment_count"] != num_gc_issue_comments: - print "GitHub issue has fewer comments than Google Code's. Fixing..." - for idx in range(last_gh_issue["comment_count"], num_gc_issue_comments): - comment = last_gc_issue.GetComments()[idx] - googlecode_comment = GoogleCodeComment(last_gc_issue, comment) - self._issue_service.CreateComment( - # issue_number == source_issue_id - int(last_gc_issue.GetId()), int(last_gc_issue.GetId()), - googlecode_comment, self._project_name) - - print "Done! Resuming normal issue execution." + # If there are existing issues, then confirm they exactly match the Google + # Code issues. Otherwise issue IDs will not match and/or there may be + # missing data. + self._AssertInGoodState() self._issue_total = len(self._issue_json_data) self._issue_number = 0 @@ -661,3 +621,59 @@ def Start(self): if self._skipped_issues > 0: print ("\nSkipped %d/%d issue previously uploaded." % (self._skipped_issues, self._issue_total)) + + def _AssertInGoodState(self): + """Checks if the last issue exported is sound, otherwise raises an error. + + Checks the existing issues that have been exported and confirms that it + matches the issue on Google Code. (Both Title and ID match.) It then + confirms that it has all of the expected comments, adding any missing ones + as necessary. + """ + if len(self._previously_created_issues) == 0: + return + + print ("Existing issues detected for the repo. Likely due to a previous\n" + "run being aborted or killed. Checking consistency...") + + # Get the last exported issue, and its dual on Google Code. + last_gh_issue_id = -1 + for id in self._previously_created_issues: + if id > last_gh_issue_id: + last_gh_issue_id = id + last_gh_issue = self._previously_created_issues[last_gh_issue_id] + + print "last_gh_issue %s %s" % (last_gh_issue_id, last_gh_issue) + print "issue_jason__daata %s" % self._issue_json_data + + last_gc_issue = None + for issue in self._issue_json_data: + if int(issue["id"]) == int(last_gh_issue_id) and ( + issue["title"] == last_gh_issue["title"]): + last_gc_issue = GoogleCodeIssue(issue, + self._project_name, + self._user_map) + break + + if last_gc_issue is None: + raise RuntimeError( + "Unable to find Google Code issue #%s '%s'.\n" + "Were issues added to GitHub since last export attempt?" % ( + last_gh_issue_id, last_gh_issue["title"])) + + print "Last issue (#%s) matches. Checking comments..." % (last_gh_issue_id) + + # Check comments. Add any missing ones as needed. + num_gc_issue_comments = len(last_gc_issue.GetComments()) + if last_gh_issue["comment_count"] != num_gc_issue_comments: + print "GitHub issue has fewer comments than Google Code's. Fixing..." + for idx in range(last_gh_issue["comment_count"], num_gc_issue_comments): + comment = last_gc_issue.GetComments()[idx] + googlecode_comment = GoogleCodeComment(last_gc_issue, comment) + # issue_number == source_issue_id + self._issue_service.CreateComment( + int(last_gc_issue.GetId()), int(last_gc_issue.GetId()), + googlecode_comment, self._project_name) + print " Added comment #%s." % (idx) + + print "Done! Issue tracker now in expected state. Ready for more exports." From 7c48af41cf52893a2835f88b84b9b1a7c4dde2ec Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 11:56:16 -0700 Subject: [PATCH 25/34] Clean up formatting --- googlecode-issues-exporter/issues.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index 67a264b..9fad90d 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -521,13 +521,16 @@ def _GetAllPreviousIssues(self): closed_issues = self._issue_service.GetIssues("closed") issues = open_issues + closed_issues for issue in issues: - print "%s" % issue - issue["id"] not in self._previously_created_issues or die( + # Yes, GitHub's issues API has both ID and Number, and they are + # the opposite of what you think they are. + issue["number"] not in self._previously_created_issues or die( "GitHub returned multiple issues with the same ID?") - self._previously_created_issues[issue["id"]] = { + self._previously_created_issues[issue["number"]] = { "title": issue["title"], "comment_count": issue["comments"], } + print "Added previous GitHub issue %s '%s' (%s comments)" % ( + issue["id"], issue["title"], issue["comments"]) def _UpdateProgressBar(self): """Update issue count 'feed'. @@ -584,6 +587,8 @@ def Start(self): This will traverse the issues and attempt to create each issue and its comments. """ + print "Starting issue export for '%s'" % (self._project_name) + # If there are existing issues, then confirm they exactly match the Google # Code issues. Otherwise issue IDs will not match and/or there may be # missing data. @@ -634,7 +639,7 @@ def _AssertInGoodState(self): return print ("Existing issues detected for the repo. Likely due to a previous\n" - "run being aborted or killed. Checking consistency...") + " run being aborted or killed. Checking consistency...") # Get the last exported issue, and its dual on Google Code. last_gh_issue_id = -1 @@ -643,9 +648,6 @@ def _AssertInGoodState(self): last_gh_issue_id = id last_gh_issue = self._previously_created_issues[last_gh_issue_id] - print "last_gh_issue %s %s" % (last_gh_issue_id, last_gh_issue) - print "issue_jason__daata %s" % self._issue_json_data - last_gc_issue = None for issue in self._issue_json_data: if int(issue["id"]) == int(last_gh_issue_id) and ( @@ -658,7 +660,7 @@ def _AssertInGoodState(self): if last_gc_issue is None: raise RuntimeError( "Unable to find Google Code issue #%s '%s'.\n" - "Were issues added to GitHub since last export attempt?" % ( + " Were issues added to GitHub since last export attempt?" % ( last_gh_issue_id, last_gh_issue["title"])) print "Last issue (#%s) matches. Checking comments..." % (last_gh_issue_id) @@ -674,6 +676,6 @@ def _AssertInGoodState(self): self._issue_service.CreateComment( int(last_gc_issue.GetId()), int(last_gc_issue.GetId()), googlecode_comment, self._project_name) - print " Added comment #%s." % (idx) + print " Added comment #%s." % (idx + 1) print "Done! Issue tracker now in expected state. Ready for more exports." From 7ce1be73f7b23966f0a5e54d42c9420077860dbd Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 12:34:19 -0700 Subject: [PATCH 26/34] Trying to clean up comments --- .../github_issue_converter_test.py | 9 +++-- googlecode-issues-exporter/issues.py | 40 ++++++++++++++----- googlecode-issues-exporter/issues_test.py | 7 ++++ 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/googlecode-issues-exporter/github_issue_converter_test.py b/googlecode-issues-exporter/github_issue_converter_test.py index 643d4b5..3499489 100644 --- a/googlecode-issues-exporter/github_issue_converter_test.py +++ b/googlecode-issues-exporter/github_issue_converter_test.py @@ -337,8 +337,8 @@ def testCloseIssue(self): def testCreateComment(self): comment_body = ( - "Comment [#1](https://code.google.com/p/repo/issues/detail" + - "?id=1#c1) originally posted by a_uthor on last year:\n\none") + "```\none\n```\n\nOriginal issue reported on code.google.com " + "by `a_uthor` on last year") self.github_issue_service.CreateComment( 1, "1", SINGLE_COMMENT, GITHUB_REPO) self.assertEqual(self.http_mock.last_method, "POST") @@ -383,6 +383,7 @@ def setUp(self): self.TEST_ISSUE_DATA = [ { "id": "1", + "number": "1", "title": "Title1", "state": "open", "comments": { @@ -395,6 +396,7 @@ def setUp(self): }, { "id": "2", + "number": "2", "title": "Title2", "state": "closed", "owner": {"kind": "projecthosting#issuePerson", @@ -407,6 +409,7 @@ def setUp(self): }, { "id": "3", + "number": "3", "title": "Title3", "state": "closed", "comments": { @@ -421,7 +424,7 @@ def setUp(self): def testGetAllPreviousIssues(self): self.assertEqual(0, len(self.issue_exporter._previously_created_issues)) - content = [{"id": 1, "title": "issue_title", "comments": 2}] + content = [{"number": 1, "title": "issue_title", "comments": 2}] self.github_service.AddResponse(content=content) self.issue_exporter._GetAllPreviousIssues() self.assertEqual(1, len(self.issue_exporter._previously_created_issues)) diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index 9fad90d..9ff572f 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -16,11 +16,23 @@ """ import collections +import datetime import json import re import sys +def TryFormatDate(date): + """Attempt to clean up a timestamp date.""" + try: + if date.endswith(":"): + date = date[:len(date) - 1] + datetime_version = datetime.datetime.strptime( + date, "%Y-%m-%dT%H:%M:%S.%fZ") + return str(datetime_version) + except ValueError as ve: + return date + class Error(Exception): """Base error class.""" @@ -335,23 +347,31 @@ def GetDescription(self): The description has been modified to include origin details. """ - source_issue_id = self.GetIssue().GetId() - project_name = self.GetIssue().GetProjectName() author = self.GetAuthor() comment_date = self.GetCreatedOn() - comment_id = self.GetId() comment_text = self.GetContent() + if not comment_text: comment_text = '<empty>' else: comment_text = FixUpComment(comment_text) - orig_comment_url = ( - "https://code.google.com/p/%s/issues/detail?id=%s#c%s" % - (project_name, source_issue_id, comment_id)) - body = "Comment [#%s](%s) originally posted by %s on %s:\n\n%s" % ( - comment_id, orig_comment_url, author, comment_date, comment_text) - return body + # Remove tags, which Codesite automatically includes if issue body is + # based on a prompt. + comment_text = comment_text.replace("", "") + comment_text = comment_text.replace("", "") + + # TODO(chrsmith): Confirm FixUpComment replaces s and such. + # TODO(chrsmith): Unescample HTML. + # TODO(chrsmith): Wrap lines at 80 chars. + body = "```\n" + comment_text + "\n```" + + # TODO(chris): Include other details, such as label adjustments. + footer = "\n\nOriginal issue reported on code.google.com by `%s` on %s" % ( + author, TryFormatDate(comment_date)) + + + return body + footer class IssueService(object): @@ -529,8 +549,6 @@ def _GetAllPreviousIssues(self): "title": issue["title"], "comment_count": issue["comments"], } - print "Added previous GitHub issue %s '%s' (%s comments)" % ( - issue["id"], issue["title"], issue["comments"]) def _UpdateProgressBar(self): """Update issue count 'feed'. diff --git a/googlecode-issues-exporter/issues_test.py b/googlecode-issues-exporter/issues_test.py index 29a805b..4712c7e 100644 --- a/googlecode-issues-exporter/issues_test.py +++ b/googlecode-issues-exporter/issues_test.py @@ -99,6 +99,13 @@ def testGetIssueUserOwner(self): issue_json, REPO, USER_MAP) self.assertEqual(DEFAULT_USERNAME, issue.GetOwner()) + def testTryFormatDate(self): + self.assertEqual("last year", issues.TryFormatDate("last year")) + self.assertEqual("2007-02-03 05:58:17", + issues.TryFormatDate("2007-02-03T05:58:17.000Z:")) + self.assertEqual("2014-01-05 04:43:15", + issues.TryFormatDate("2014-01-05T04:43:15.000Z")) + if __name__ == "__main__": unittest.main(buffer=True) From 7e353e38d2fe4b274a032e495daa7caa1b02e707 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 12:36:03 -0700 Subject: [PATCH 27/34] Stuff --- googlecode-issues-exporter/bitbucket_issue_converter_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/googlecode-issues-exporter/bitbucket_issue_converter_test.py b/googlecode-issues-exporter/bitbucket_issue_converter_test.py index 3147173..85e091c 100644 --- a/googlecode-issues-exporter/bitbucket_issue_converter_test.py +++ b/googlecode-issues-exporter/bitbucket_issue_converter_test.py @@ -87,8 +87,8 @@ def testCloseIssue(self): def testCreateComment(self): comment_body = { "content": ( - "Comment [#1](https://code.google.com/p/repo/issues/detail" + - "?id=1#c1) originally posted by a_uthor on last year:\n\none"), + "```\none\n```\n\nOriginal issue reported on code.google.com " + "by `a_uthor` on last year"), "created_on": "last year", "id": 1, "issue": 1, From 21741aea0ad4b8f213d150546de5489a9a71c171 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 12:51:05 -0700 Subject: [PATCH 28/34] Add word wrap. --- googlecode-issues-exporter/issues.py | 50 +++++++++++------------ googlecode-issues-exporter/issues_test.py | 8 ++++ 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index 9ff572f..0b67a41 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -33,6 +33,26 @@ def TryFormatDate(date): except ValueError as ve: return date + +def WrapText(text, max): + """Inserts a newline if any line of a file is > max chars. + + Note that the newline is inserted at the first whitespace + character, so there may be lines longer than max. + """ + char_list = list(text) + last_linebreak = 0 + for i in range(0, len(char_list)): + if char_list[i] == '\n': + last_linebreak = i + if i - last_linebreak > max and char_list[i] == ' ': + # Replace ' ' with '\n' + char_list.pop(i) + char_list.insert(i, '\n') + last_linebreak = i + return ''.join(char_list) + + class Error(Exception): """Base error class.""" @@ -233,7 +253,6 @@ def GetDescription(self): # description. googlecode_comment = GoogleCodeComment(self, self._GetDescription()) content = googlecode_comment.GetContent() - content = FixUpComment(content) author = googlecode_comment.GetAuthor() create_date = googlecode_comment.GetCreatedOn() url = "https://code.google.com/p/%s/issues/detail?id=%s" % ( @@ -352,17 +371,14 @@ def GetDescription(self): comment_text = self.GetContent() if not comment_text: - comment_text = '<empty>' - else: - comment_text = FixUpComment(comment_text) + comment_text = "(No text was entered with this change)" # Remove tags, which Codesite automatically includes if issue body is # based on a prompt. comment_text = comment_text.replace("", "") comment_text = comment_text.replace("", "") - # TODO(chrsmith): Confirm FixUpComment replaces s and such. - # TODO(chrsmith): Unescample HTML. + # TODO(chrsmith): Unescample HTML. e.g. > and á # TODO(chrsmith): Wrap lines at 80 chars. body = "```\n" + comment_text + "\n```" @@ -429,26 +445,6 @@ def CreateComment(self, issue_number, source_issue_id, raise NotImplementedError() -def FixUpComment(comment): - """Fixes up comments.""" - formatted = [] - preformat_rest_of_comment = False - for line in comment.split("\n"): - if re.match(r'^#+ ', line) or re.match(r'^Index: ', line): - preformat_rest_of_comment = True - elif '--- cut here ---' in line: - preformat_rest_of_comment = True - if preformat_rest_of_comment: - formatted.append(" %s" % line) - else: - # "#3" style commends get converted into links to issue #3, etc. - # We don't want this. There's no way to escape this so put a non - # breaking space to prevent. - line = re.sub(r"#(\d+)", r"# \g<1>", line) - formatted.append(line) - return '\n'.join(formatted) - - def LoadIssueData(issue_file_path, project_name): """Loads issue data from a file. @@ -686,7 +682,7 @@ def _AssertInGoodState(self): # Check comments. Add any missing ones as needed. num_gc_issue_comments = len(last_gc_issue.GetComments()) if last_gh_issue["comment_count"] != num_gc_issue_comments: - print "GitHub issue has fewer comments than Google Code's. Fixing..." + print "GitHub issue has fewer comments than Google Code's. Fixng..." for idx in range(last_gh_issue["comment_count"], num_gc_issue_comments): comment = last_gc_issue.GetComments()[idx] googlecode_comment = GoogleCodeComment(last_gc_issue, comment) diff --git a/googlecode-issues-exporter/issues_test.py b/googlecode-issues-exporter/issues_test.py index 4712c7e..a0aad2e 100644 --- a/googlecode-issues-exporter/issues_test.py +++ b/googlecode-issues-exporter/issues_test.py @@ -106,6 +106,14 @@ def testTryFormatDate(self): self.assertEqual("2014-01-05 04:43:15", issues.TryFormatDate("2014-01-05T04:43:15.000Z")) + def testWrapText(self): + self.assertEqual(issues.WrapText("0123456789", 3), + "0123456789") + self.assertEqual(issues.WrapText("01234 56789", 3), + "01234\n56789") + self.assertEqual(issues.WrapText("a b c d e f g h", 4), + "a b c\nd e f\ng h") + if __name__ == "__main__": unittest.main(buffer=True) From 5359ece4623e110d601d4912028869e780a5bcc1 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 12:54:03 -0700 Subject: [PATCH 29/34] Wire in WrapText --- googlecode-issues-exporter/issues.py | 1 + 1 file changed, 1 insertion(+) diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index 0b67a41..3d07115 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -377,6 +377,7 @@ def GetDescription(self): # based on a prompt. comment_text = comment_text.replace("", "") comment_text = comment_text.replace("", "") + comment_text = WrapText(comment_text, 80) # TODO(chrsmith): Unescample HTML. e.g. > and á # TODO(chrsmith): Wrap lines at 80 chars. From ccc3222acefba9e830477a0fb2073ee4fc1fbb45 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 12:58:58 -0700 Subject: [PATCH 30/34] Fix attachment generation --- .../github_issue_converter.py | 2 +- googlecode-issues-exporter/issues.py | 63 +++++++++---------- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/googlecode-issues-exporter/github_issue_converter.py b/googlecode-issues-exporter/github_issue_converter.py index 4a16091..91ee480 100644 --- a/googlecode-issues-exporter/github_issue_converter.py +++ b/googlecode-issues-exporter/github_issue_converter.py @@ -172,7 +172,7 @@ def PerformPostRequest(self, url, body): # only applies to certain API calls (like creating issues). And, alas, the # exact quota is undocumented. So the value below is simply a guess. See: # https://developer.github.com/v3/#abuse-rate-limits - req_min = 25 + req_min = 15 time.sleep(60 / req_min) return self._PerformHttpRequest("POST", url, body) diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index 3d07115..e3d0530 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -284,37 +284,7 @@ def GetContent(self): Returns: The issue comment """ - body = self._comment["content"] - - # Add references to attachments as appropriate. - attachmentLines = [] - for attachment in self._comment["attachments"] if "attachments" in self._comment else []: - if "isDeleted" in attachment: - line = " * *Attachment: %s (deleted)*" % (attachment["fileName"]) - attachmentLines.append(line) - continue - - link = "https://storage.googleapis.com/google-code-attachments/%s/issue-%d/comment-%d/%s" % ( - self.GetIssue().GetProjectName(), self.GetIssue().GetId(), - self.GetId(), attachment["fileName"]) - - def has_extension(extension): - return attachment["fileName"].lower().endswith(extension) - - is_image_attachment = False - for extension in [".png", ".jpg", ".jpeg", ".bmp", ".tif", ".gif"]: - is_image_attachment |= has_extension(".png") - - if is_image_attachment: - line = " * *Attachment: %s
![%s](%s)*" % ( - attachment["fileName"], attachment["fileName"], link) - else: - line = " * *Attachment: [%s](%s)*" % (attachment["fileName"], link) - attachmentLines.append(line) - - if len(attachmentLines) > 0: - body += "\n
\n" + "\n".join(attachmentLines) - return body + return self._comment["content"] def GetCreatedOn(self): """Get the creation date from a Google Code comment. @@ -377,7 +347,7 @@ def GetDescription(self): # based on a prompt. comment_text = comment_text.replace("", "") comment_text = comment_text.replace("", "") - comment_text = WrapText(comment_text, 80) + comment_text = WrapText(comment_text, 82) # In case it was already wrapped... # TODO(chrsmith): Unescample HTML. e.g. > and á # TODO(chrsmith): Wrap lines at 80 chars. @@ -387,7 +357,36 @@ def GetDescription(self): footer = "\n\nOriginal issue reported on code.google.com by `%s` on %s" % ( author, TryFormatDate(comment_date)) + # Add references to attachments as appropriate. + attachmentLines = [] + for attachment in self._comment["attachments"] if "attachments" in self._comment else []: + if "isDeleted" in attachment: + line = " * *Attachment: %s (deleted)*" % (attachment["fileName"]) + attachmentLines.append(line) + continue + + link = "https://storage.googleapis.com/google-code-attachments/%s/issue-%d/comment-%d/%s" % ( + self.GetIssue().GetProjectName(), self.GetIssue().GetId(), + self.GetId(), attachment["fileName"]) + + def has_extension(extension): + return attachment["fileName"].lower().endswith(extension) + + is_image_attachment = False + for extension in [".png", ".jpg", ".jpeg", ".bmp", ".tif", ".gif"]: + is_image_attachment |= has_extension(".png") + + if is_image_attachment: + line = " * *Attachment: %s
![%s](%s)*" % ( + attachment["fileName"], attachment["fileName"], link) + else: + line = " * *Attachment: [%s](%s)*" % (attachment["fileName"], link) + attachmentLines.append(line) + + if len(attachmentLines) > 0: + footer += "\n
\n" + "\n".join(attachmentLines) + # Return the data to send to generate the comment. return body + footer From e0bd584b1054a5d3e8dc48692bc6d0087aaf4b24 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 15:58:37 -0700 Subject: [PATCH 31/34] Remove 'default issue author' --- .../bitbucket_issue_converter.py | 9 +++------ .../github_issue_converter.py | 3 +-- googlecode-issues-exporter/issues.py | 16 +++++++++++----- googlecode-issues-exporter/issues_test.py | 8 ++++++++ 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/googlecode-issues-exporter/bitbucket_issue_converter.py b/googlecode-issues-exporter/bitbucket_issue_converter.py index 508a2a1..65ede2b 100644 --- a/googlecode-issues-exporter/bitbucket_issue_converter.py +++ b/googlecode-issues-exporter/bitbucket_issue_converter.py @@ -172,14 +172,14 @@ def WriteIssueData(self, default_issue_kind): def ExportIssues(issue_file_path, project_name, - user_file_path, default_issue_kind, default_username): + user_file_path, default_issue_kind): """Exports all issues for a given project. """ issue_service = IssueService() user_service = UserService() issue_data = issues.LoadIssueData(issue_file_path, project_name) - user_map = issues.LoadUserData(user_file_path, default_username, user_service) + user_map = issues.LoadUserData(user_file_path, user_service) issue_exporter = issues.IssueExporter( issue_service, user_service, issue_data, project_name, user_map) @@ -218,8 +218,6 @@ def main(args): help="A non-null string containing one of the following" "values: bug, enhancement, proposal, task. Defaults to" "bug") - parser.add_argument("--default_owner_username", required=True, - help="The default issue owner's username") parsed_args, _ = parser.parse_known_args(args) # Default value. @@ -229,8 +227,7 @@ def main(args): ExportIssues( parsed_args.issue_file_path, parsed_args.project_name, - parsed_args.user_file_path, parsed_args.default_issue_kind, - parsed_args.default_owner_username) + parsed_args.user_file_path, parsed_args.default_issue_kind) if __name__ == "__main__": diff --git a/googlecode-issues-exporter/github_issue_converter.py b/googlecode-issues-exporter/github_issue_converter.py index 91ee480..ffd4098 100644 --- a/googlecode-issues-exporter/github_issue_converter.py +++ b/googlecode-issues-exporter/github_issue_converter.py @@ -415,8 +415,7 @@ def ExportIssues(github_owner_username, github_repo_name, github_oauth_token, user_service = UserService(github_service) issue_data = issues.LoadIssueData(issue_file_path, project_name) - user_map = issues.LoadUserData( - user_file_path, github_owner_username, user_service) + user_map = issues.LoadUserData(user_file_path, user_service) issue_exporter = issues.IssueExporter( issue_service, user_service, issue_data, project_name, user_map) diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index e3d0530..12075e7 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -22,6 +22,11 @@ import sys +class IdentityDict(dict): + def __missing__(self, key): + return key + + def TryFormatDate(date): """Attempt to clean up a timestamp date.""" try: @@ -325,6 +330,7 @@ def GetAuthor(self): The Google Code username that the issue comment is authored by or the repository owner if no mapping or email address exists. """ + print "GetAuthor" if "author" not in self._comment: return None @@ -469,17 +475,17 @@ def LoadIssueData(issue_file_path, project_name): raise ProjectNotFoundError("Project %s not found" % project_name) -def LoadUserData(user_file_path, default_username, user_service): - """Loads user data from a file. +def LoadUserData(user_file_path, user_service): + """Loads user data from a file. If not present, the user name will + just return whatever is passed to it. Args: user_file_path: path to the file to load - default_username: Username to use when no entry is found for a user. user_service: an instance of UserService """ - result = collections.defaultdict(lambda: default_username) + identity_dict = IdentityDict() if not user_file_path: - return result + return identity_dict with open(user_file_path) as user_data: user_json = user_data.read() diff --git a/googlecode-issues-exporter/issues_test.py b/googlecode-issues-exporter/issues_test.py index a0aad2e..b073792 100644 --- a/googlecode-issues-exporter/issues_test.py +++ b/googlecode-issues-exporter/issues_test.py @@ -99,6 +99,9 @@ def testGetIssueUserOwner(self): issue_json, REPO, USER_MAP) self.assertEqual(DEFAULT_USERNAME, issue.GetOwner()) + def testGetCommentAuthor(self): + self.assertEqual("a_uthor", SINGLE_COMMENT.GetAuthor()) + def testTryFormatDate(self): self.assertEqual("last year", issues.TryFormatDate("last year")) self.assertEqual("2007-02-03 05:58:17", @@ -114,6 +117,11 @@ def testWrapText(self): self.assertEqual(issues.WrapText("a b c d e f g h", 4), "a b c\nd e f\ng h") + def testLoadUserData(self): + # Verify the "identity dictionary" behavior. + user_data_dict = issues.LoadUserData(None, None) + self.assertEqual(user_data_dict["chrs...@goog.com"], "chrs...@goog.com") + if __name__ == "__main__": unittest.main(buffer=True) From 07ba936125a13259967682cd1d9bc707e1db7b09 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 16:09:57 -0700 Subject: [PATCH 32/34] Unify Issue.GetDescription and Comment.GetDescription --- .../bitbucket_issue_converter_test.py | 5 ++-- .../github_issue_converter_test.py | 5 ++-- googlecode-issues-exporter/issues.py | 23 ++++--------------- 3 files changed, 10 insertions(+), 23 deletions(-) diff --git a/googlecode-issues-exporter/bitbucket_issue_converter_test.py b/googlecode-issues-exporter/bitbucket_issue_converter_test.py index 85e091c..fa87938 100644 --- a/googlecode-issues-exporter/bitbucket_issue_converter_test.py +++ b/googlecode-issues-exporter/bitbucket_issue_converter_test.py @@ -63,8 +63,9 @@ def setUp(self): def testCreateIssue(self): issue_body = { "assignee": "a_uthor", - "content": ("Original [issue 1](https://code.google.com/p/repo/issues" + - "/detail?id=1) created by a_uthor on last year:\n\none"), + "content": ( + "```\none\n```\n\nOriginal issue reported on code.google.com" + " by `a_uthor` on last year"), "content_updated_on": "last month", "created_on": "last year", "id": 1, diff --git a/googlecode-issues-exporter/github_issue_converter_test.py b/googlecode-issues-exporter/github_issue_converter_test.py index 3499489..03170c7 100644 --- a/googlecode-issues-exporter/github_issue_converter_test.py +++ b/googlecode-issues-exporter/github_issue_converter_test.py @@ -311,8 +311,9 @@ def setUp(self): def testCreateIssue(self): issue_body = { - "body": ("Original [issue 1](https://code.google.com/p/repo/issues" + - "/detail?id=1) created by a_uthor on last year:\n\none"), + "body": ( + "```\none\n```\n\nOriginal issue reported on code.google.com" + " by `a_uthor` on last year"), "assignee": "a_uthor", "labels": ["awesome", "great"], "title": "issue_title", diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index 12075e7..ef5e3db 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -249,22 +249,10 @@ def IsOpen(self): return "state" in self._issue and self._issue["state"] == "open" def GetDescription(self): - """Returns the Description of the issue. - - The description has been modified to include origin details. - """ - issue_id = self.GetId() - # code.google.com always has one comment (item #0) which is the issue - # description. + """Returns the Description of the issue.""" + # Just return the description of the underlying comment. googlecode_comment = GoogleCodeComment(self, self._GetDescription()) - content = googlecode_comment.GetContent() - author = googlecode_comment.GetAuthor() - create_date = googlecode_comment.GetCreatedOn() - url = "https://code.google.com/p/%s/issues/detail?id=%s" % ( - self._project_name, issue_id) - body = "Original [issue %s](%s) created by %s on %s:\n\n%s" % ( - issue_id, url, author, create_date, content) - return body + return googlecode_comment.GetDescription() class GoogleCodeComment(object): @@ -338,10 +326,7 @@ def GetAuthor(self): return self.GetIssue().GetUserMap()[author] def GetDescription(self): - """Returns the Description of the comment. - - The description has been modified to include origin details. - """ + """Returns the Description of the comment.""" author = self.GetAuthor() comment_date = self.GetCreatedOn() comment_text = self.GetContent() From f559f55191ffcf74ef6d03fc99ed6552b9ba4d75 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 16:31:44 -0700 Subject: [PATCH 33/34] Fix bugs, add hack for issue ownership --- .../github_issue_converter.py | 16 +++++++++++----- googlecode-issues-exporter/issues.py | 10 ++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/googlecode-issues-exporter/github_issue_converter.py b/googlecode-issues-exporter/github_issue_converter.py index ffd4098..6672b86 100644 --- a/googlecode-issues-exporter/github_issue_converter.py +++ b/googlecode-issues-exporter/github_issue_converter.py @@ -307,7 +307,7 @@ def GetIssues(self, state="open"): response, content = self._github_service.PerformGetRequest( self._github_issues_url, params=params) if not _CheckSuccessful(response): - raise IOError("Failed to retrieve previous issues.\n%s" % content) + raise IOError("Failed to retrieve previous issues.\n\n%s" % content) if not content: return github_issues else: @@ -343,8 +343,10 @@ def CreateIssue(self, googlecode_issue): if not _CheckSuccessful(response): # Newline character at the beginning of the line to allows for in-place # updating of the counts of the issues and comments. - raise issues.ServiceError("\nFailed to create issue %d: %s.\n%s" % ( - googlecode_issue.GetId(), issue_title, content)) + raise issues.ServiceError( + "\nFailed to create issue #%d '%s'.\n\n\n" + "Response:\n%s\n\n\nContent:\n%s" % ( + googlecode_issue.GetId(), issue_title, response, content)) return self._GetIssueNumber(content) @@ -387,8 +389,9 @@ def CreateComment(self, issue_number, source_issue_id, if not _CheckSuccessful(response): raise issues.ServiceError( - "\nFailed to create issue comment (%s) for issue #%d\n%s" % - (googlecode_comment.GetContent(), issue_number, content)) + "\nFailed to create issue comment for issue #%d\n\n" + "Response:\n%s\n\nContent:\n%s\n\n" % + (issue_number, response, content)) time.sleep(self._comment_delay) def _GetIssueNumber(self, content): @@ -417,6 +420,9 @@ def ExportIssues(github_owner_username, github_repo_name, github_oauth_token, issue_data = issues.LoadIssueData(issue_file_path, project_name) user_map = issues.LoadUserData(user_file_path, user_service) + # Add a special "user_requesting_export" user, which comes in handy. + user_map["user_requesting_export"] = github_owner_username + issue_exporter = issues.IssueExporter( issue_service, user_service, issue_data, project_name, user_map) diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index ef5e3db..23cd0bd 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -121,14 +121,9 @@ def GetUserMap(self): def GetOwner(self): """Get the owner username of a Google Code issue. - Returns: - The Google Code username that owns the issue or the - repository owner if no mapping or email address exists. + This will ALWAYS be the person requesting the issue export. """ - if "owner" not in self._issue: - return None - owner = self._issue["owner"]["name"] - return self._user_map[owner] + return self._user_map["user_requesting_export"] def GetContentUpdatedOn(self): """Get the date the content was last updated from a Google Code issue. @@ -318,7 +313,6 @@ def GetAuthor(self): The Google Code username that the issue comment is authored by or the repository owner if no mapping or email address exists. """ - print "GetAuthor" if "author" not in self._comment: return None From 59c5324af2794e1148cb72972156a634bd7c59e4 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 25 Apr 2015 17:41:48 -0700 Subject: [PATCH 34/34] Wire in labels --- .../bitbucket_issue_converter_test.py | 11 +++++-- .../github_issue_converter_test.py | 11 ++++--- googlecode-issues-exporter/issues.py | 30 +++++++++++++++---- googlecode-issues-exporter/issues_test.py | 17 +++++++++-- 4 files changed, 55 insertions(+), 14 deletions(-) diff --git a/googlecode-issues-exporter/bitbucket_issue_converter_test.py b/googlecode-issues-exporter/bitbucket_issue_converter_test.py index fa87938..7b6b008 100644 --- a/googlecode-issues-exporter/bitbucket_issue_converter_test.py +++ b/googlecode-issues-exporter/bitbucket_issue_converter_test.py @@ -62,10 +62,12 @@ def setUp(self): def testCreateIssue(self): issue_body = { - "assignee": "a_uthor", + "assignee": "default_username", "content": ( "```\none\n```\n\nOriginal issue reported on code.google.com" - " by `a_uthor` on last year"), + " by `a_uthor` on last year\n" + "- **Labels added**: added-label\n" + "- **Labels removed**: removed-label\n"), "content_updated_on": "last month", "created_on": "last year", "id": 1, @@ -85,11 +87,14 @@ def testCloseIssue(self): # no-op self._bitbucket_issue_service.CloseIssue(123) + # TODO(chris): Add testcase for an issue comment with attachments. def testCreateComment(self): comment_body = { "content": ( "```\none\n```\n\nOriginal issue reported on code.google.com " - "by `a_uthor` on last year"), + "by `a_uthor` on last year\n" + "- **Labels added**: added-label\n" + "- **Labels removed**: removed-label\n"), "created_on": "last year", "id": 1, "issue": 1, diff --git a/googlecode-issues-exporter/github_issue_converter_test.py b/googlecode-issues-exporter/github_issue_converter_test.py index 03170c7..5d6e667 100644 --- a/googlecode-issues-exporter/github_issue_converter_test.py +++ b/googlecode-issues-exporter/github_issue_converter_test.py @@ -312,9 +312,10 @@ def setUp(self): def testCreateIssue(self): issue_body = { "body": ( - "```\none\n```\n\nOriginal issue reported on code.google.com" - " by `a_uthor` on last year"), - "assignee": "a_uthor", + "```\none\n```\n\nOriginal issue reported on code.google.com by `a_uthor` on last year\n" + "- **Labels added**: added-label\n" + "- **Labels removed**: removed-label\n"), + "assignee": "default_username", "labels": ["awesome", "great"], "title": "issue_title", } @@ -339,7 +340,9 @@ def testCloseIssue(self): def testCreateComment(self): comment_body = ( "```\none\n```\n\nOriginal issue reported on code.google.com " - "by `a_uthor` on last year") + "by `a_uthor` on last year\n" + "- **Labels added**: added-label\n" + "- **Labels removed**: removed-label\n") self.github_issue_service.CreateComment( 1, "1", SINGLE_COMMENT, GITHUB_REPO) self.assertEqual(self.http_mock.last_method, "POST") diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index 23cd0bd..fd498bc 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -290,6 +290,13 @@ def GetId(self): """ return self._comment["id"] + def GetLabels(self): + """Get the labels modified with the comment.""" + if "updates" in self._comment: + if "labels" in self._comment["updates"]: + return self._comment["updates"]["labels"] + return [] + def GetIssue(self): """Get the GoogleCodeIssue this comment belongs to. @@ -330,24 +337,37 @@ def GetDescription(self): # Remove tags, which Codesite automatically includes if issue body is # based on a prompt. + # TODO(chrsmith): Unescample HTML. e.g. > and á comment_text = comment_text.replace("", "") comment_text = comment_text.replace("", "") comment_text = WrapText(comment_text, 82) # In case it was already wrapped... - # TODO(chrsmith): Unescample HTML. e.g. > and á - # TODO(chrsmith): Wrap lines at 80 chars. body = "```\n" + comment_text + "\n```" - # TODO(chris): Include other details, such as label adjustments. footer = "\n\nOriginal issue reported on code.google.com by `%s` on %s" % ( author, TryFormatDate(comment_date)) + # Add label adjustments. + if self.GetLabels(): + labels_added = [] + labels_removed = [] + for label in self.GetLabels(): + if label.startswith("-"): + labels_removed.append(label[1:]) + else: + labels_added.append(label) + + footer += "\n" + if labels_added: + footer += "- **Labels added**: %s\n" % (", ".join(labels_added)) + if labels_removed: + footer += "- **Labels removed**: %s\n" % (", ".join(labels_removed)) + # Add references to attachments as appropriate. attachmentLines = [] for attachment in self._comment["attachments"] if "attachments" in self._comment else []: if "isDeleted" in attachment: - line = " * *Attachment: %s (deleted)*" % (attachment["fileName"]) - attachmentLines.append(line) + # Deleted attachments won't be found on the issue mirror. continue link = "https://storage.googleapis.com/google-code-attachments/%s/issue-%d/comment-%d/%s" % ( diff --git a/googlecode-issues-exporter/issues_test.py b/googlecode-issues-exporter/issues_test.py index b073792..74278a9 100644 --- a/googlecode-issues-exporter/issues_test.py +++ b/googlecode-issues-exporter/issues_test.py @@ -40,6 +40,9 @@ "id": 1, "published": "last year", "author": {"name": "user@email.com"}, + "updates": { + "labels": ["added-label", "-removed-label"], + }, } COMMENT_TWO = { "content": "two", @@ -84,13 +87,15 @@ class GoogleCodeIssueTest(unittest.TestCase): """Tests for GoogleCodeIssue.""" def testGetIssueOwner(self): - self.assertEqual("a_uthor", SINGLE_ISSUE.GetOwner()) + # Report all issues coming from the person who initiated the + # export. + self.assertEqual("default_username", SINGLE_ISSUE.GetOwner()) def testGetIssueOwnerNoOwner(self): issue_json = ISSUE_JSON.copy() del issue_json["owner"] issue = issues.GoogleCodeIssue(issue_json, REPO, USER_MAP) - self.assertEqual(None, issue.GetOwner()) + self.assertEqual("default_username", issue.GetOwner()) def testGetIssueUserOwner(self): issue_json = copy.deepcopy(ISSUE_JSON) @@ -102,6 +107,14 @@ def testGetIssueUserOwner(self): def testGetCommentAuthor(self): self.assertEqual("a_uthor", SINGLE_COMMENT.GetAuthor()) + def testGetCommentDescription(self): + self.assertEqual( + "```\none\n```\n\nOriginal issue reported on code.google.com by " + "`a_uthor` on last year\n" + "- **Labels added**: added-label\n" + "- **Labels removed**: removed-label\n", + SINGLE_COMMENT.GetDescription()) + def testTryFormatDate(self): self.assertEqual("last year", issues.TryFormatDate("last year")) self.assertEqual("2007-02-03 05:58:17",