diff --git a/googlecode-issues-exporter/bitbucket_issue_converter.py b/googlecode-issues-exporter/bitbucket_issue_converter.py index b718646..65ede2b 100644 --- a/googlecode-issues-exporter/bitbucket_issue_converter.py +++ b/googlecode-issues-exporter/bitbucket_issue_converter.py @@ -28,6 +28,44 @@ import issues +def _getKind(kind): + mapping = { + "defect": "bug", + "enhancement": "enhancement", + "task": "task", + "review": "proposal", + "other": "bug", + } + return mapping.get(kind.lower(), "bug") + + +def _getPriority(priority): + mapping = { + "low": "trivial", + "medium": "minor", + "high": "major", + "critical": "critical", + } + return mapping.get(priority.lower(), "minor") + + +def _getStatus(status): + mapping = { + "new": "new", + "fixed": "resolved", + "invalid": "invalid", + "duplicate": "duplicate", + "wontfix": "wontfix", + } + return mapping.get(status.lower(), "new") + + +def _getTitle(title): + if len(title) < 255: + return title + return title[:250] + "[...]" + + class UserService(issues.UserService): """BitBucket user operations. """ @@ -80,11 +118,11 @@ def CreateIssue(self, googlecode_issue): "content_updated_on": googlecode_issue.GetContentUpdatedOn(), "created_on": googlecode_issue.GetCreatedOn(), "id": googlecode_issue.GetId(), - "kind": googlecode_issue.GetKind(), - "priority": googlecode_issue.GetPriority(), + "kind": _getKind(googlecode_issue.GetKind()), + "priority": _getPriority(googlecode_issue.GetPriority()), "reporter": googlecode_issue.GetAuthor(), - "status": googlecode_issue.GetStatus(), - "title": googlecode_issue.GetTitle(), + "status": _getStatus(googlecode_issue.GetStatus()), + "title": _getTitle(googlecode_issue.GetTitle()), "updated_on": googlecode_issue.GetUpdatedOn() } self._bitbucket_issues.append(bitbucket_issue) @@ -129,19 +167,19 @@ def WriteIssueData(self, default_issue_kind): } with open("db-1.0.json", "w") as issues_file: issues_json = json.dumps(issues_data, sort_keys=True, indent=4, - separators=(",", ": "), ensure_ascii=False) - issues_file.write(unicode(issues_json)) + separators=(",", ": ")) + issues_file.write(issues_json) def ExportIssues(issue_file_path, project_name, - user_file_path, default_issue_kind, default_username): + user_file_path, default_issue_kind): """Exports all issues for a given project. """ issue_service = IssueService() user_service = UserService() issue_data = issues.LoadIssueData(issue_file_path, project_name) - user_map = issues.LoadUserData(user_file_path, default_username, user_service) + user_map = issues.LoadUserData(user_file_path, user_service) issue_exporter = issues.IssueExporter( issue_service, user_service, issue_data, project_name, user_map) @@ -173,21 +211,23 @@ def main(args): parser.add_argument("--project_name", required=True, help="The name of the Google Code project you wish to" "export") - parser.add_argument("--user_file_path", required=True, + parser.add_argument("--user_file_path", required=False, help="The path to the file containing a mapping from" "email address to bitbucket username") parser.add_argument("--default_issue_kind", required=False, help="A non-null string containing one of the following" "values: bug, enhancement, proposal, task. Defaults to" - "bug.") - parser.add_argument("--default_owner_username", required=True, - help="The default issue username") + "bug") parsed_args, _ = parser.parse_known_args(args) + # Default value. + if not parsed_args.default_issue_kind: + print "Using default issue kind of 'bug'." + parsed_args.default_issue_kind = "bug" + ExportIssues( parsed_args.issue_file_path, parsed_args.project_name, - parsed_args.user_file_path, parsed_args.default_issue_kind, - parsed_args.default_owner_username) + parsed_args.user_file_path, parsed_args.default_issue_kind) if __name__ == "__main__": diff --git a/googlecode-issues-exporter/bitbucket_issue_converter_test.py b/googlecode-issues-exporter/bitbucket_issue_converter_test.py index a2f5de4..7b6b008 100644 --- a/googlecode-issues-exporter/bitbucket_issue_converter_test.py +++ b/googlecode-issues-exporter/bitbucket_issue_converter_test.py @@ -62,16 +62,19 @@ def setUp(self): def testCreateIssue(self): issue_body = { - "assignee": "a_uthor", - "content": ("Original [issue 1](https://code.google.com/p/repo/issues" + - "/detail?id=1) created by a_uthor on last year:\n\none"), + "assignee": "default_username", + "content": ( + "```\none\n```\n\nOriginal issue reported on code.google.com" + " by `a_uthor` on last year\n" + "- **Labels added**: added-label\n" + "- **Labels removed**: removed-label\n"), "content_updated_on": "last month", "created_on": "last year", "id": 1, - "kind": "Defect", - "priority": "Medium", + "kind": "bug", + "priority": "minor", "reporter": None, - "status": "fixed", + "status": "resolved", "title": "issue_title", "updated_on": "last year", } @@ -84,11 +87,14 @@ def testCloseIssue(self): # no-op self._bitbucket_issue_service.CloseIssue(123) + # TODO(chris): Add testcase for an issue comment with attachments. def testCreateComment(self): comment_body = { "content": ( - "Comment [#1](https://code.google.com/p/repo/issues/detail" + - "?id=1#c1) originally posted by a_uthor on last year:\n\none"), + "```\none\n```\n\nOriginal issue reported on code.google.com " + "by `a_uthor` on last year\n" + "- **Labels added**: added-label\n" + "- **Labels removed**: removed-label\n"), "created_on": "last year", "id": 1, "issue": 1, diff --git a/googlecode-issues-exporter/github_issue_converter.py b/googlecode-issues-exporter/github_issue_converter.py index 1b30b82..6672b86 100644 --- a/googlecode-issues-exporter/github_issue_converter.py +++ b/googlecode-issues-exporter/github_issue_converter.py @@ -89,10 +89,11 @@ class GitHubService(object): Attributes: github_owner_username: The username of the owner of the repository. github_repo_name: The GitHub repository name. + rate_limit: Whether or not to rate limit API calls. """ def __init__(self, github_owner_username, github_repo_name, - github_oauth_token, http_instance=None): + github_oauth_token, rate_limit, http_instance=None): """Initialize the GitHubService. Args: @@ -105,6 +106,7 @@ def __init__(self, github_owner_username, github_repo_name, self.github_owner_username = github_owner_username self.github_repo_name = github_repo_name self._github_oauth_token = github_oauth_token + self._rate_limit = rate_limit self._http = http_instance if http_instance else httplib2.Http() def _PerformHttpRequest(self, method, url, body="{}", params=None): @@ -124,13 +126,15 @@ def _PerformHttpRequest(self, method, url, body="{}", params=None): A tuple of an HTTP response (https://developer.github.com/v3/#schema) and its content from the server which is decoded JSON. """ + headers = { "User-Agent": "GoogleCodeIssueExporter/1.0" } query = params.copy() if params else {} query["access_token"] = self._github_oauth_token request_url = "%s%s?%s" % (GITHUB_API_URL, url, urllib.urlencode(query)) requests = 0 while requests < MAX_HTTP_REQUESTS: requests += 1 - response, content = self._http.request(request_url, method, body) + response, content = self._http.request(request_url, method, + headers=headers, body=body) if _CheckSuccessful(response): return response, json.loads(content) elif self._RequestLimitReached(): @@ -162,13 +166,14 @@ def PerformPostRequest(self, url, body): A tuple of an HTTP response (https://developer.github.com/v3/#schema) and its content from the server which is decoded JSON. """ - # Add a delay to all outgoing request to GitHub, as to not trigger their - # anti-abuse mechanism. This is separate from your typical rate limit, and - # only applies to certain API calls (like creating issues). And, alas, the - # exact quota is undocumented. So the value below is simply a guess. See: - # https://developer.github.com/v3/#abuse-rate-limits - req_min = 30 - time.sleep(60 / req_min) + if self._rate_limit and self._rate_limit in ["True", "true"]: + # Add a delay to all outgoing request to GitHub, as to not trigger their + # anti-abuse mechanism. This is separate from your typical rate limit, and + # only applies to certain API calls (like creating issues). And, alas, the + # exact quota is undocumented. So the value below is simply a guess. See: + # https://developer.github.com/v3/#abuse-rate-limits + req_min = 15 + time.sleep(60 / req_min) return self._PerformHttpRequest("POST", url, body) def PerformPatchRequest(self, url, body): @@ -268,7 +273,6 @@ class IssueService(issues.IssueService): def __init__(self, github_service, comment_delay=COMMENT_DELAY): """Initialize the IssueService. - Args: github_service: The GitHub service. """ @@ -303,7 +307,7 @@ def GetIssues(self, state="open"): response, content = self._github_service.PerformGetRequest( self._github_issues_url, params=params) if not _CheckSuccessful(response): - raise IOError("Failed to retrieve previous issues.\n%s" % content) + raise IOError("Failed to retrieve previous issues.\n\n%s" % content) if not content: return github_issues else: @@ -323,6 +327,10 @@ def CreateIssue(self, googlecode_issue): issues.ServiceError: An error occurred creating the issue. """ issue_title = googlecode_issue.GetTitle() + # It is not possible to create a Google Code issue without a title, but you + # can edit an issue to remove its title afterwards. + if issue_title.isspace(): + issue_title = "" issue = { "title": issue_title, "body": googlecode_issue.GetDescription(), @@ -335,8 +343,10 @@ def CreateIssue(self, googlecode_issue): if not _CheckSuccessful(response): # Newline character at the beginning of the line to allows for in-place # updating of the counts of the issues and comments. - raise issues.ServiceError("\nFailed to create issue: %s.\n%s" % ( - issue_title, content)) + raise issues.ServiceError( + "\nFailed to create issue #%d '%s'.\n\n\n" + "Response:\n%s\n\n\nContent:\n%s" % ( + googlecode_issue.GetId(), issue_title, response, content)) return self._GetIssueNumber(content) @@ -379,8 +389,9 @@ def CreateComment(self, issue_number, source_issue_id, if not _CheckSuccessful(response): raise issues.ServiceError( - "\nFailed to create issue comment (%s) for issue #%d\n%s" % - (googlecode_comment.GetContent(), issue_number, content)) + "\nFailed to create issue comment for issue #%d\n\n" + "Response:\n%s\n\nContent:\n%s\n\n" % + (issue_number, response, content)) time.sleep(self._comment_delay) def _GetIssueNumber(self, content): @@ -392,22 +403,25 @@ def _GetIssueNumber(self, content): Returns: The GitHub issue number. """ - assert "number" in content, "nope %s" % content + assert "number" in content, "Getting issue number from: %s" % content return content["number"] def ExportIssues(github_owner_username, github_repo_name, github_oauth_token, - issue_file_path, project_name, user_file_path): + issue_file_path, project_name, user_file_path, rate_limit): """Exports all issues for a given project. """ github_service = GitHubService( - github_owner_username, github_repo_name, github_oauth_token) + github_owner_username, github_repo_name, github_oauth_token, + rate_limit) issue_service = IssueService(github_service) user_service = UserService(github_service) issue_data = issues.LoadIssueData(issue_file_path, project_name) - user_map = issues.LoadUserData( - user_file_path, github_owner_username, user_service) + user_map = issues.LoadUserData(user_file_path, user_service) + + # Add a special "user_requesting_export" user, which comes in handy. + user_map["user_requesting_export"] = github_owner_username issue_exporter = issues.IssueExporter( issue_service, user_service, issue_data, project_name, user_map) @@ -449,12 +463,16 @@ def main(args): parser.add_argument("--user_file_path", required=False, help="The path to the file containing a mapping from" "email address to github username.") + parser.add_argument("--rate_limit", required=False, default="True", + help="Rate limit GitHub requests to not run into" + "anti-abuse limits.") parsed_args, _ = parser.parse_known_args(args) ExportIssues( parsed_args.github_owner_username, parsed_args.github_repo_name, parsed_args.github_oauth_token, parsed_args.issue_file_path, - parsed_args.project_name, parsed_args.user_file_path) + parsed_args.project_name, parsed_args.user_file_path, + parsed_args.rate_limit) if __name__ == "__main__": diff --git a/googlecode-issues-exporter/github_issue_converter_test.py b/googlecode-issues-exporter/github_issue_converter_test.py index 365ca13..5d6e667 100644 --- a/googlecode-issues-exporter/github_issue_converter_test.py +++ b/googlecode-issues-exporter/github_issue_converter_test.py @@ -166,7 +166,7 @@ def __init__(self): self.last_method = None self.last_body = None - def request(self, url, method, body=None): + def request(self, url, method, headers=None, body=None): """Makes a fake HTTP request. Args: @@ -190,6 +190,7 @@ def setUp(self): self.http_mock = Http2Mock() self.github_service = github_issue_converter.GitHubService( GITHUB_USERNAME, GITHUB_REPO, GITHUB_TOKEN, + rate_limit=False, http_instance=self.http_mock) def testSuccessfulRequestSuccess(self): @@ -303,15 +304,18 @@ def setUp(self): self.http_mock = Http2Mock() self.github_service = github_issue_converter.GitHubService( GITHUB_USERNAME, GITHUB_REPO, GITHUB_TOKEN, + rate_limit=False, http_instance=self.http_mock) self.github_issue_service = github_issue_converter.IssueService( self.github_service, comment_delay=0) def testCreateIssue(self): issue_body = { - "body": ("Original [issue 1](https://code.google.com/p/repo/issues" + - "/detail?id=1) created by a_uthor on last year:\n\none"), - "assignee": "a_uthor", + "body": ( + "```\none\n```\n\nOriginal issue reported on code.google.com by `a_uthor` on last year\n" + "- **Labels added**: added-label\n" + "- **Labels removed**: removed-label\n"), + "assignee": "default_username", "labels": ["awesome", "great"], "title": "issue_title", } @@ -335,8 +339,10 @@ def testCloseIssue(self): def testCreateComment(self): comment_body = ( - "Comment [#1](https://code.google.com/p/repo/issues/detail" + - "?id=1#c1) originally posted by a_uthor on last year:\n\none") + "```\none\n```\n\nOriginal issue reported on code.google.com " + "by `a_uthor` on last year\n" + "- **Labels added**: added-label\n" + "- **Labels removed**: removed-label\n") self.github_issue_service.CreateComment( 1, "1", SINGLE_COMMENT, GITHUB_REPO) self.assertEqual(self.http_mock.last_method, "POST") @@ -378,14 +384,59 @@ def setUp(self): NO_ISSUE_DATA, GITHUB_REPO, USER_MAP) self.issue_exporter.Init() + self.TEST_ISSUE_DATA = [ + { + "id": "1", + "number": "1", + "title": "Title1", + "state": "open", + "comments": { + "items": [COMMENT_ONE, COMMENT_TWO, COMMENT_THREE], + }, + "labels": ["Type-Issue", "Priority-High"], + "owner": {"kind": "projecthosting#issuePerson", + "name": "User1" + }, + }, + { + "id": "2", + "number": "2", + "title": "Title2", + "state": "closed", + "owner": {"kind": "projecthosting#issuePerson", + "name": "User2" + }, + "labels": [], + "comments": { + "items": [COMMENT_ONE], + }, + }, + { + "id": "3", + "number": "3", + "title": "Title3", + "state": "closed", + "comments": { + "items": [COMMENT_ONE, COMMENT_TWO], + }, + "labels": ["Type-Defect"], + "owner": {"kind": "projecthosting#issuePerson", + "name": "User3" + } + }] + + def testGetAllPreviousIssues(self): self.assertEqual(0, len(self.issue_exporter._previously_created_issues)) - content = [{"title": "issue_title"}] + content = [{"number": 1, "title": "issue_title", "comments": 2}] self.github_service.AddResponse(content=content) self.issue_exporter._GetAllPreviousIssues() self.assertEqual(1, len(self.issue_exporter._previously_created_issues)) - self.assertTrue("issue_title" in - self.issue_exporter._previously_created_issues) + self.assertTrue(1 in self.issue_exporter._previously_created_issues) + self.assertEqual("issue_title", + self.issue_exporter._previously_created_issues[1]["title"]) + self.assertEqual(2, + self.issue_exporter._previously_created_issues[1]["comment_count"]) def testCreateIssue(self): content = {"number": 1234} @@ -418,52 +469,16 @@ def testCreateCommentsFailure(self): self.issue_exporter._CreateComments(COMMENTS_DATA, 1234, SINGLE_ISSUE) def testStart(self): - self.issue_exporter._issue_json_data = [ - { - "id": "1", - "title": "Title1", - "state": "open", - "comments": { - "items": [COMMENT_ONE, COMMENT_TWO, COMMENT_THREE], - }, - "labels": ["Type-Issue", "Priority-High"], - "owner": {"kind": "projecthosting#issuePerson", - "name": "User1" - }, - }, - { - "id": "2", - "title": "Title2", - "state": "closed", - "owner": {"kind": "projecthosting#issuePerson", - "name": "User2" - }, - "labels": [], - "comments": { - "items": [COMMENT_ONE], - }, - }, - { - "id": "3", - "title": "Title3", - "state": "closed", - "comments": { - "items": [COMMENT_ONE, COMMENT_TWO], - }, - "labels": ["Type-Defect"], - "owner": {"kind": "projecthosting#issuePerson", - "name": "User3" - } - }] + self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA - self.github_service.AddResponse(content={"number": 1234}) - self.github_service.AddResponse(content={"number": 2345}) - self.github_service.AddResponse(content={"number": 3456}) - self.github_service.AddResponse(content={"number": 5678}) - self.github_service.AddResponse(content={"number": 6789}) - self.github_service.AddResponse(content={"number": 7890}) - self.github_service.AddResponse(content={"number": 8901}) - self.github_service.AddResponse(content={"number": 9012}) + # Note: Some responses are from CreateIssues, others are from CreateComment. + self.github_service.AddResponse(content={"number": 1}) + self.github_service.AddResponse(content={"number": 10}) + self.github_service.AddResponse(content={"number": 11}) + self.github_service.AddResponse(content={"number": 2}) + self.github_service.AddResponse(content={"number": 20}) + self.github_service.AddResponse(content={"number": 3}) + self.github_service.AddResponse(content={"number": 30}) self.issue_exporter.Start() @@ -475,12 +490,68 @@ def testStart(self): self.assertEqual(1, self.issue_exporter._comment_number) self.assertEqual(1, self.issue_exporter._comment_total) - def testStartSkipAlreadyCreatedIssues(self): - self.issue_exporter._previously_created_issues.add("Title1") - self.issue_exporter._issue_json_data = [{"title": "Title1"}] + def testStart_SkipAlreadyCreatedIssues(self): + self.issue_exporter._previously_created_issues["1"] = { + "title": "Title1", + "comment_count": 3 + } + self.issue_exporter._previously_created_issues["2"] = { + "title": "Title2", + "comment_count": 1 + } + self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA + + self.github_service.AddResponse(content={"number": 3}) + self.issue_exporter.Start() - self.assertEqual(1, self.issue_exporter._issue_total) - self.assertEqual(0, self.issue_exporter._issue_number) + + self.assertEqual(2, self.issue_exporter._skipped_issues) + self.assertEqual(3, self.issue_exporter._issue_total) + self.assertEqual(3, self.issue_exporter._issue_number) + + def testStart_ReAddMissedComments(self): + self.issue_exporter._previously_created_issues["1"] = { + "title": "Title1", + "comment_count": 1 # Missing 2 comments. + } + self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA + + # First requests to re-add comments, then create issues. + self.github_service.AddResponse(content={"number": 11}) + + self.github_service.AddResponse(content={"number": 2}) + self.github_service.AddResponse(content={"number": 20}) + self.github_service.AddResponse(content={"number": 3}) + self.github_service.AddResponse(content={"number": 30}) + + self.issue_exporter.Start() + self.assertEqual(1, self.issue_exporter._skipped_issues) + self.assertEqual(3, self.issue_exporter._issue_total) + self.assertEqual(3, self.issue_exporter._issue_number) + + + def testStart_GetErrorIfGoogleCodeAndGitHubDoNotMatch(self): + self.issue_exporter._previously_created_issues["1"] = { + "title": "Title1"} + self.issue_exporter._previously_created_issues["2"] = { + "title": "< not issue #2's title >"} + self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA + + with self.assertRaises(RuntimeError): + self.issue_exporter.Start() + + def testStart_GetErrorIfCreatedGitHubIDDoesNotMatch(self): + self.issue_exporter._issue_json_data = self.TEST_ISSUE_DATA + + # Note: Some responses are from CreateIssues, others are from CreateComment. + self.github_service.AddResponse(content={"number": 1}) + self.github_service.AddResponse(content={"number": 10}) + self.github_service.AddResponse(content={"number": 11}) + self.github_service.AddResponse(content={"number": 3}) # Expects next issue ID 2. + + with self.assertRaises(RuntimeError): + self.issue_exporter.Start() + if __name__ == "__main__": unittest.main(buffer=True) diff --git a/googlecode-issues-exporter/issues.py b/googlecode-issues-exporter/issues.py index c0fef13..fd498bc 100644 --- a/googlecode-issues-exporter/issues.py +++ b/googlecode-issues-exporter/issues.py @@ -16,11 +16,48 @@ """ import collections +import datetime import json import re import sys +class IdentityDict(dict): + def __missing__(self, key): + return key + + +def TryFormatDate(date): + """Attempt to clean up a timestamp date.""" + try: + if date.endswith(":"): + date = date[:len(date) - 1] + datetime_version = datetime.datetime.strptime( + date, "%Y-%m-%dT%H:%M:%S.%fZ") + return str(datetime_version) + except ValueError as ve: + return date + + +def WrapText(text, max): + """Inserts a newline if any line of a file is > max chars. + + Note that the newline is inserted at the first whitespace + character, so there may be lines longer than max. + """ + char_list = list(text) + last_linebreak = 0 + for i in range(0, len(char_list)): + if char_list[i] == '\n': + last_linebreak = i + if i - last_linebreak > max and char_list[i] == ' ': + # Replace ' ' with '\n' + char_list.pop(i) + char_list.insert(i, '\n') + last_linebreak = i + return ''.join(char_list) + + class Error(Exception): """Base error class.""" @@ -84,14 +121,9 @@ def GetUserMap(self): def GetOwner(self): """Get the owner username of a Google Code issue. - Returns: - The Google Code username that owns the issue or the - repository owner if no mapping or email address exists. + This will ALWAYS be the person requesting the issue export. """ - if "owner" not in self._issue: - return None - owner = self._issue["owner"]["name"] - return self._user_map[owner] + return self._user_map["user_requesting_export"] def GetContentUpdatedOn(self): """Get the date the content was last updated from a Google Code issue. @@ -123,7 +155,7 @@ def GetLabels(self): Returns: A list of the labels of this issue. """ - return self._issue["labels"] + return self._issue.get("labels", []) def GetKind(self): """Get the kind from a Google Code issue. @@ -212,23 +244,10 @@ def IsOpen(self): return "state" in self._issue and self._issue["state"] == "open" def GetDescription(self): - """Returns the Description of the issue. - - The description has been modified to include origin details. - """ - issue_id = self.GetId() - # code.google.com always has one comment (item #0) which is the issue - # description. + """Returns the Description of the issue.""" + # Just return the description of the underlying comment. googlecode_comment = GoogleCodeComment(self, self._GetDescription()) - content = googlecode_comment.GetContent() - content = FixUpComment(content) - author = googlecode_comment.GetAuthor() - create_date = googlecode_comment.GetCreatedOn() - url = "https://code.google.com/p/%s/issues/detail?id=%s" % ( - self._project_name, issue_id) - body = "Original [issue %s](%s) created by %s on %s:\n\n%s" % ( - issue_id, url, author, create_date, content) - return body + return googlecode_comment.GetDescription() class GoogleCodeComment(object): @@ -271,6 +290,13 @@ def GetId(self): """ return self._comment["id"] + def GetLabels(self): + """Get the labels modified with the comment.""" + if "updates" in self._comment: + if "labels" in self._comment["updates"]: + return self._comment["updates"]["labels"] + return [] + def GetIssue(self): """Get the GoogleCodeIssue this comment belongs to. @@ -301,27 +327,72 @@ def GetAuthor(self): return self.GetIssue().GetUserMap()[author] def GetDescription(self): - """Returns the Description of the comment. - - The description has been modified to include origin details. - """ - source_issue_id = self.GetIssue().GetId() - project_name = self.GetIssue().GetProjectName() + """Returns the Description of the comment.""" author = self.GetAuthor() comment_date = self.GetCreatedOn() - comment_id = self.GetId() comment_text = self.GetContent() + if not comment_text: - comment_text = '<empty>' - else: - comment_text = FixUpComment(comment_text) + comment_text = "(No text was entered with this change)" + + # Remove tags, which Codesite automatically includes if issue body is + # based on a prompt. + # TODO(chrsmith): Unescample HTML. e.g. > and á + comment_text = comment_text.replace("", "") + comment_text = comment_text.replace("", "") + comment_text = WrapText(comment_text, 82) # In case it was already wrapped... + + body = "```\n" + comment_text + "\n```" + + footer = "\n\nOriginal issue reported on code.google.com by `%s` on %s" % ( + author, TryFormatDate(comment_date)) + + # Add label adjustments. + if self.GetLabels(): + labels_added = [] + labels_removed = [] + for label in self.GetLabels(): + if label.startswith("-"): + labels_removed.append(label[1:]) + else: + labels_added.append(label) + + footer += "\n" + if labels_added: + footer += "- **Labels added**: %s\n" % (", ".join(labels_added)) + if labels_removed: + footer += "- **Labels removed**: %s\n" % (", ".join(labels_removed)) + + # Add references to attachments as appropriate. + attachmentLines = [] + for attachment in self._comment["attachments"] if "attachments" in self._comment else []: + if "isDeleted" in attachment: + # Deleted attachments won't be found on the issue mirror. + continue + + link = "https://storage.googleapis.com/google-code-attachments/%s/issue-%d/comment-%d/%s" % ( + self.GetIssue().GetProjectName(), self.GetIssue().GetId(), + self.GetId(), attachment["fileName"]) + + def has_extension(extension): + return attachment["fileName"].lower().endswith(extension) + + is_image_attachment = False + for extension in [".png", ".jpg", ".jpeg", ".bmp", ".tif", ".gif"]: + is_image_attachment |= has_extension(".png") + + if is_image_attachment: + line = " * *Attachment: %s
![%s](%s)*" % ( + attachment["fileName"], attachment["fileName"], link) + else: + line = " * *Attachment: [%s](%s)*" % (attachment["fileName"], link) + attachmentLines.append(line) - orig_comment_url = ( - "https://code.google.com/p/%s/issues/detail?id=%s#c%s" % - (project_name, source_issue_id, comment_id)) - body = "Comment [#%s](%s) originally posted by %s on %s:\n\n%s" % ( - comment_id, orig_comment_url, author, comment_date, comment_text) - return body + if len(attachmentLines) > 0: + footer += "\n
\n" + "\n".join(attachmentLines) + + # Return the data to send to generate the comment. + return body + footer class IssueService(object): @@ -379,26 +450,6 @@ def CreateComment(self, issue_number, source_issue_id, raise NotImplementedError() -def FixUpComment(comment): - """Fixes up comments.""" - formatted = [] - preformat_rest_of_comment = False - for line in comment.split("\n"): - if re.match(r'^#+ ', line) or re.match(r'^Index: ', line): - preformat_rest_of_comment = True - elif '--- cut here ---' in line: - preformat_rest_of_comment = True - if preformat_rest_of_comment: - formatted.append(" %s" % line) - else: - # "#3" style commends get converted into links to issue #3, etc. - # We don't want this. There's no way to escape this so put a non - # breaking space to prevent. - line = re.sub(r"#(\d+)", r"# \g<1>", line) - formatted.append(line) - return '\n'.join(formatted) - - def LoadIssueData(issue_file_path, project_name): """Loads issue data from a file. @@ -423,17 +474,17 @@ def LoadIssueData(issue_file_path, project_name): raise ProjectNotFoundError("Project %s not found" % project_name) -def LoadUserData(user_file_path, default_username, user_service): - """Loads user data from a file. +def LoadUserData(user_file_path, user_service): + """Loads user data from a file. If not present, the user name will + just return whatever is passed to it. Args: user_file_path: path to the file to load - default_username: Username to use when no entry is found for a user. user_service: an instance of UserService """ - result = collections.defaultdict(lambda: default_username) + identity_dict = IdentityDict() if not user_file_path: - return result + return identity_dict with open(user_file_path) as user_data: user_json = user_data.read() @@ -470,29 +521,35 @@ def __init__(self, issue_service, user_service, issue_json_data, self._project_name = project_name self._user_map = user_map - self._previously_created_issues = set() + # Mapping from issue ID to the issue's metadata. This is used to verify + # consistency with an previous attempts at exporting issues. + self._previously_created_issues = {} self._issue_total = 0 self._issue_number = 0 self._comment_number = 0 self._comment_total = 0 + self._skipped_issues = 0 def Init(self): """Initialize the needed variables.""" self._GetAllPreviousIssues() def _GetAllPreviousIssues(self): - """Gets all previously uploaded issues. - - Creates a hash of the issue titles, they will be unique as the Google Code - issue number is in each title. - """ + """Gets all previously uploaded issues.""" print "Getting any previously added issues..." open_issues = self._issue_service.GetIssues("open") closed_issues = self._issue_service.GetIssues("closed") issues = open_issues + closed_issues for issue in issues: - self._previously_created_issues.add(issue["title"]) + # Yes, GitHub's issues API has both ID and Number, and they are + # the opposite of what you think they are. + issue["number"] not in self._previously_created_issues or die( + "GitHub returned multiple issues with the same ID?") + self._previously_created_issues[issue["number"]] = { + "title": issue["title"], + "comment_count": issue["comments"], + } def _UpdateProgressBar(self): """Update issue count 'feed'. @@ -549,28 +606,34 @@ def Start(self): This will traverse the issues and attempt to create each issue and its comments. """ - if len(self._previously_created_issues): - print ("Existing issues detected for the repo. Likely due to" - " the script being previously aborted or killed.") + print "Starting issue export for '%s'" % (self._project_name) + + # If there are existing issues, then confirm they exactly match the Google + # Code issues. Otherwise issue IDs will not match and/or there may be + # missing data. + self._AssertInGoodState() self._issue_total = len(self._issue_json_data) self._issue_number = 0 - skipped_issues = 0 + self._skipped_issues = 0 for issue in self._issue_json_data: googlecode_issue = GoogleCodeIssue( issue, self._project_name, self._user_map) - issue_title = googlecode_issue.GetTitle() self._issue_number += 1 self._UpdateProgressBar() - if issue_title in self._previously_created_issues: - skipped_issues += 1 + issue_id = googlecode_issue.GetId() + if issue_id in self._previously_created_issues: + self._skipped_issues += 1 continue issue_number = self._CreateIssue(googlecode_issue) if issue_number < 0: continue + if int(issue_number) != int(googlecode_issue.GetId()): + raise RuntimeError("Google Code and GitHub issue nos mismatch at #%s (got %s)" % ( + googlecode_issue.GetId(), issue_number)) comments = googlecode_issue.GetComments() self._CreateComments(comments, issue_number, googlecode_issue) @@ -578,6 +641,60 @@ def Start(self): if not googlecode_issue.IsOpen(): self._issue_service.CloseIssue(issue_number) - if skipped_issues > 0: + # TODO(chrsmith): Issue a warning if/when the issue ID get out of sync. + if self._skipped_issues > 0: print ("\nSkipped %d/%d issue previously uploaded." % - (skipped_issues, self._issue_total)) + (self._skipped_issues, self._issue_total)) + + def _AssertInGoodState(self): + """Checks if the last issue exported is sound, otherwise raises an error. + + Checks the existing issues that have been exported and confirms that it + matches the issue on Google Code. (Both Title and ID match.) It then + confirms that it has all of the expected comments, adding any missing ones + as necessary. + """ + if len(self._previously_created_issues) == 0: + return + + print ("Existing issues detected for the repo. Likely due to a previous\n" + " run being aborted or killed. Checking consistency...") + + # Get the last exported issue, and its dual on Google Code. + last_gh_issue_id = -1 + for id in self._previously_created_issues: + if id > last_gh_issue_id: + last_gh_issue_id = id + last_gh_issue = self._previously_created_issues[last_gh_issue_id] + + last_gc_issue = None + for issue in self._issue_json_data: + if int(issue["id"]) == int(last_gh_issue_id) and ( + issue["title"] == last_gh_issue["title"]): + last_gc_issue = GoogleCodeIssue(issue, + self._project_name, + self._user_map) + break + + if last_gc_issue is None: + raise RuntimeError( + "Unable to find Google Code issue #%s '%s'.\n" + " Were issues added to GitHub since last export attempt?" % ( + last_gh_issue_id, last_gh_issue["title"])) + + print "Last issue (#%s) matches. Checking comments..." % (last_gh_issue_id) + + # Check comments. Add any missing ones as needed. + num_gc_issue_comments = len(last_gc_issue.GetComments()) + if last_gh_issue["comment_count"] != num_gc_issue_comments: + print "GitHub issue has fewer comments than Google Code's. Fixng..." + for idx in range(last_gh_issue["comment_count"], num_gc_issue_comments): + comment = last_gc_issue.GetComments()[idx] + googlecode_comment = GoogleCodeComment(last_gc_issue, comment) + # issue_number == source_issue_id + self._issue_service.CreateComment( + int(last_gc_issue.GetId()), int(last_gc_issue.GetId()), + googlecode_comment, self._project_name) + print " Added comment #%s." % (idx + 1) + + print "Done! Issue tracker now in expected state. Ready for more exports." diff --git a/googlecode-issues-exporter/issues_test.py b/googlecode-issues-exporter/issues_test.py index 29a805b..74278a9 100644 --- a/googlecode-issues-exporter/issues_test.py +++ b/googlecode-issues-exporter/issues_test.py @@ -40,6 +40,9 @@ "id": 1, "published": "last year", "author": {"name": "user@email.com"}, + "updates": { + "labels": ["added-label", "-removed-label"], + }, } COMMENT_TWO = { "content": "two", @@ -84,13 +87,15 @@ class GoogleCodeIssueTest(unittest.TestCase): """Tests for GoogleCodeIssue.""" def testGetIssueOwner(self): - self.assertEqual("a_uthor", SINGLE_ISSUE.GetOwner()) + # Report all issues coming from the person who initiated the + # export. + self.assertEqual("default_username", SINGLE_ISSUE.GetOwner()) def testGetIssueOwnerNoOwner(self): issue_json = ISSUE_JSON.copy() del issue_json["owner"] issue = issues.GoogleCodeIssue(issue_json, REPO, USER_MAP) - self.assertEqual(None, issue.GetOwner()) + self.assertEqual("default_username", issue.GetOwner()) def testGetIssueUserOwner(self): issue_json = copy.deepcopy(ISSUE_JSON) @@ -99,6 +104,37 @@ def testGetIssueUserOwner(self): issue_json, REPO, USER_MAP) self.assertEqual(DEFAULT_USERNAME, issue.GetOwner()) + def testGetCommentAuthor(self): + self.assertEqual("a_uthor", SINGLE_COMMENT.GetAuthor()) + + def testGetCommentDescription(self): + self.assertEqual( + "```\none\n```\n\nOriginal issue reported on code.google.com by " + "`a_uthor` on last year\n" + "- **Labels added**: added-label\n" + "- **Labels removed**: removed-label\n", + SINGLE_COMMENT.GetDescription()) + + def testTryFormatDate(self): + self.assertEqual("last year", issues.TryFormatDate("last year")) + self.assertEqual("2007-02-03 05:58:17", + issues.TryFormatDate("2007-02-03T05:58:17.000Z:")) + self.assertEqual("2014-01-05 04:43:15", + issues.TryFormatDate("2014-01-05T04:43:15.000Z")) + + def testWrapText(self): + self.assertEqual(issues.WrapText("0123456789", 3), + "0123456789") + self.assertEqual(issues.WrapText("01234 56789", 3), + "01234\n56789") + self.assertEqual(issues.WrapText("a b c d e f g h", 4), + "a b c\nd e f\ng h") + + def testLoadUserData(self): + # Verify the "identity dictionary" behavior. + user_data_dict = issues.LoadUserData(None, None) + self.assertEqual(user_data_dict["chrs...@goog.com"], "chrs...@goog.com") + if __name__ == "__main__": unittest.main(buffer=True) diff --git a/wiki-to-md/convert-repo.sh b/wiki_to_md/convert-repo.sh old mode 100644 new mode 100755 similarity index 100% rename from wiki-to-md/convert-repo.sh rename to wiki_to_md/convert-repo.sh diff --git a/wiki-to-md/example.md b/wiki_to_md/example.md similarity index 84% rename from wiki-to-md/example.md rename to wiki_to_md/example.md index 3682b00..38f497b 100644 --- a/wiki-to-md/example.md +++ b/wiki_to_md/example.md @@ -1,6 +1,20 @@ # Header1 -(TODO: Add table of contents.) +This is a test file for verifying the tool's output. You can regenerate +`example.md` yourself by running: + +``` +python wiki2gfm.py \ + --project=test \ + --wikipages_list="TestPage" \ + --input_file=example.wiki \ + --output_file=example.md +``` + +Note that this is used by {{wiki2gfm\_test.py}}, so changing the +arguments will break unit tests. + + #### Tables (mistmatched header sides) @@ -12,22 +26,22 @@ Tables: |**Name/Sample** | **Markup** | |:-----------------|:-----------------| -| _italic_ | ` _italic_ ` | -| **bold** | ` *bold* ` | -| ` code ` | `` `code` `` | -| ` code ` | ` {{{code}}} ` | -| superscript | ` ^super^script ` | -| subscript | ` ,,sub,,script ` | -| ~~strikeout~~ | ` ~~strikeout~~ ` | +| _italic_ | `_italic_` | +| **bold** | `*bold*` | +| `code` | ```code``` | +| `code` | `{{{code}}}` | +| superscript | `^super^script` | +| subscript | `,,sub,,script` | +| ~~strikeout~~ | `~~strikeout~~` | You can mix these typefaces in some ways: | **Markup** | **Result** | |:------------------------------------|:----------------------------------| -| ` _*bold* in italics_ ` | _**bold** in italics_ | -| ` *_italics_ in bold* ` | **_italics_ in bold** | -| ` *~~strike~~ works too* ` | **~~strike~~ works too** | -| ` ~~as well as _this_ way round~~ ` | ~~as well as _this_ way round~~ | +| `_*bold* in italics_` | _**bold** in italics_ | +| `*_italics_ in bold*` | **_italics_ in bold** | +| `*~~strike~~ works too*` | **~~strike~~ works too** | +| `~~as well as _this_ way round~~` | ~~as well as _this_ way round~~ | **IN HTML** @@ -165,9 +179,9 @@ The following is:
==== Links -A LittleLink[?](wiki/TestPage) +A LittleLink[?](TestPage.md) -[TestPage](wiki/TestPage) is linked automatically. +[TestPage](TestPage.md) is linked automatically. http://www.google.com/ @@ -184,9 +198,9 @@ http://www.google.com/ **IN HTML** -A LittleLink? +A LittleLink? -TestPage is linked automatically.
+TestPage is linked automatically.

http://www.google.com/ @@ -231,7 +245,7 @@ rev111612.
This text will be removed from the rendered page. '> -(TODO: Link to Google+ page.) + @@ -240,7 +254,7 @@ This text will be removed from the rendered page. This text will be removed from the rendered page. '>

-(TODO: Link to Google+ page.)
+


 @@ -272,4 +286,10 @@ this is a var!
## All done! -Hope you enjoyed. \ No newline at end of file +Hope you enjoyed. + +# Regressions + +We were adding extra spaces around tick marks: + +alpha `beta` gamma diff --git a/wiki-to-md/example.wiki b/wiki_to_md/example.wiki similarity index 92% rename from wiki-to-md/example.wiki rename to wiki_to_md/example.wiki index 450c130..d647328 100644 --- a/wiki-to-md/example.wiki +++ b/wiki_to_md/example.wiki @@ -3,6 +3,20 @@ #labels Ignored. = Header1 = +This is a test file for verifying the tool's output. You can regenerate +{{{example.md}}} yourself by running: + +{{{ +python wiki2gfm.py \ + --project=test \ + --wikipages_list="TestPage" \ + --input_file=example.wiki \ + --output_file=example.md +}}} + +Note that this is used by {{wiki2gfm_test.py}}, so changing the +arguments will break unit tests. + ==== Tables (mistmatched header sides) == @@ -275,3 +289,10 @@ This text will be removed from the rendered page. == All done!== Hope you enjoyed. + += Regressions = + +We were adding extra spaces around tick marks: + +alpha `beta` gamma + diff --git a/wiki-to-md/impl/__init__.py b/wiki_to_md/impl/__init__.py similarity index 100% rename from wiki-to-md/impl/__init__.py rename to wiki_to_md/impl/__init__.py diff --git a/wiki-to-md/impl/constants.py b/wiki_to_md/impl/constants.py similarity index 100% rename from wiki-to-md/impl/constants.py rename to wiki_to_md/impl/constants.py diff --git a/wiki-to-md/impl/converter.py b/wiki_to_md/impl/converter.py similarity index 95% rename from wiki-to-md/impl/converter.py rename to wiki_to_md/impl/converter.py index 39e26c5..f1128d5 100644 --- a/wiki-to-md/impl/converter.py +++ b/wiki_to_md/impl/converter.py @@ -127,7 +127,7 @@ def Convert(self, input_stream, output_stream): if remaining_lines != 0: self._warning_method( input_line, - "Processing completed, but not all lines were processed. " + u"Processing completed, but not all lines were processed. " "Remaining lines: {0}.".format(remaining_lines)) def _ExtractPragmas(self, input_line, input_lines, output_stream): @@ -320,7 +320,7 @@ def _ProcessLine( if line[indent_pos + 1] != " ": self._warning_method( input_line, - "Missing space after list symbol: {0}, " + u"Missing space after list symbol: {0}, " "'{1}' was removed instead." .format(line[indent_pos], line[indent_pos + 1])) line = line[indent_pos + 2:] @@ -398,7 +398,7 @@ def _SetCurrentList(self, input_line, indent_pos, list_type, output_stream): else: self._warning_method( input_line, - "Bad list type: '{0}'".format(list_type)) + u"Bad list type: '{0}'".format(list_type)) return True @@ -411,11 +411,11 @@ def _OpenTag(self, input_line, tag, output_stream): output_stream: Output Markdown file. """ handler = getattr( - self._formatting_handler, "Handle{0}Open".format(tag), None) + self._formatting_handler, u"Handle{0}Open".format(tag), None) if handler: handler(input_line, output_stream) else: - self._warning_method(input_line, "Bad open tag: '{0}'".format(tag)) + self._warning_method(input_line, u"Bad open tag: '{0}'".format(tag)) self._open_tags.append(tag) @@ -428,11 +428,11 @@ def _CloseTag(self, input_line, tag, output_stream): output_stream: Output Markdown file. """ handler = getattr( - self._formatting_handler, "Handle{0}Close".format(tag), None) + self._formatting_handler, u"Handle{0}Close".format(tag), None) if handler: handler(input_line, output_stream) else: - self._warning_method(input_line, "Bad close tag: '{0}'".format(tag)) + self._warning_method(input_line, u"Bad close tag: '{0}'".format(tag)) self._open_tags.remove(tag) @@ -509,7 +509,7 @@ def _ProcessMatch(self, input_line, match_regex, line, output_stream): output_stream, match) else: - handler = getattr(self, "_Handle{0}".format(rulename), None) + handler = getattr(self, u"_Handle{0}".format(rulename), None) handler(input_line, match, output_stream) lastpos = fullmatch.end() @@ -935,13 +935,13 @@ def _HandlePlugin(self, input_line, match, output_stream): else: self._warning_method( input_line, - "Unknown plugin was given, outputting " + u"Unknown plugin was given, outputting " "as plain text:\n\t{0}".format(match)) # Wiki syntax put this class of error on its own line. self._formatting_handler.HandleEscapedText( input_line, output_stream, - "\n\n{0}\n\n".format(match)) + u"\n\n{0}\n\n".format(match)) # Add plugin and parameters to the stack. if not has_end: @@ -974,7 +974,7 @@ def _HandlePluginHtml( else: self._warning_method( input_line, - "The following parameter was given for the '{0}' tag, " + u"The following parameter was given for the '{0}' tag, " "but will not be present in the outputted HTML:\n\t'{1}': '{2}'" .format(plugin_id, name, value)) @@ -1012,7 +1012,7 @@ def _HandlePluginGPlus( else: self._warning_method( input_line, - "The following parameter was given for the '{0}' tag, " + u"The following parameter was given for the '{0}' tag, " "but will not be present in the outputted HTML:\n\t'{1}': '{2}'" .format(plugin_id, name, value)) @@ -1038,7 +1038,7 @@ def _HandlePluginWikiComment( for name, value in params.items(): self._warning_method( input_line, - "The following parameter was given for the '{0}' tag, " + u"The following parameter was given for the '{0}' tag, " "but will not be present in the outputted HTML:\n\t'{1}': '{2}'" .format(plugin_id, name, value)) @@ -1054,7 +1054,7 @@ def _HandlePluginWikiGadget(self, input_line, match, output_stream): """ self._warning_method( input_line, - "A wiki gadget was used, but this must be manually converted to a " + u"A wiki gadget was used, but this must be manually converted to a " "GFM-supported method, if possible. Outputting as plain text:\n\t{0}" .format(match)) self._formatting_handler.HandleEscapedText( @@ -1083,7 +1083,7 @@ def _HandlePluginWikiVideo( else: self._warning_method( input_line, - "The following parameter was given for the '{0}' tag, " + u"The following parameter was given for the '{0}' tag, " "but will not be present in the outputted HTML:\n\t'{1}': '{2}'" .format(plugin_id, name, value)) @@ -1100,13 +1100,13 @@ def _HandlePluginWikiVideo( "video id within parameter \"url\".") self._warning_method( input_line, - "Video plugin has invalid video ID, outputting error:\n\t{0}" + u"Video plugin has invalid video ID, outputting error:\n\t{0}" .format(output)) # Wiki syntax put this class of error on its own line. self._formatting_handler.HandleEscapedText( input_line, output_stream, - "\n\n{0}\n\n".format(output)) + u"\n\n{0}\n\n".format(output)) else: self._formatting_handler.HandleVideoOpen( input_line, @@ -1118,13 +1118,13 @@ def _HandlePluginWikiVideo( output = "wiki:video: missing mandatory parameter \"url\"." self._warning_method( input_line, - "Video plugin is missing 'url' parameter, outputting error:\n\t{0}" + u"Video plugin is missing 'url' parameter, outputting error:\n\t{0}" .format(output)) # Wiki syntax put this class of error on its own line. self._formatting_handler.HandleEscapedText( input_line, output_stream, - "\n\n{0}\n\n".format(output)) + u"\n\n{0}\n\n".format(output)) def _HandlePluginWikiToc(self, input_line, match, output_stream): """Handle a plugin tag for a wiki table of contents. @@ -1134,19 +1134,14 @@ def _HandlePluginWikiToc(self, input_line, match, output_stream): match: Matched text. output_stream: Output Markdown file. """ - output = "(TODO: Add table of contents.)" self._warning_method( input_line, - "A table of contents plugin was used for this wiki:\n" + u"A table of contents plugin was used for this wiki:\n" "\t{0}\n" "The Gollum wiki system supports table of content generation.\n" "See https://github.com/gollum/gollum/wiki for more information.\n" - "It has been replaced with:\n\t{1}" - .format(match, output)) - self._formatting_handler.HandleEscapedText( - input_line, - output_stream, - output) + "It has been removed." + .format(match)) def _HandlePluginEnd(self, input_line, match, output_stream): """Handle a plugin ending tag. @@ -1190,17 +1185,17 @@ def _HandlePluginEnd(self, input_line, match, output_stream): else: self._warning_method( input_line, - "Unknown but matching plugin end was given, outputting " + u"Unknown but matching plugin end was given, outputting " "as plain text:\n\t{0}".format(match)) # Wiki syntax put this class of error on its own line. self._formatting_handler.HandleEscapedText( input_line, output_stream, - "\n\n{0}\n\n".format(match)) + u"\n\n{0}\n\n".format(match)) else: self._warning_method( input_line, - "Unknown/unmatched plugin end was given, outputting " + u"Unknown/unmatched plugin end was given, outputting " "as plain text with errors:\n\t{0}".format(match)) # Wiki syntax put this class of error on its own line, # with a prefix error message, and did not display the tag namespace. @@ -1208,7 +1203,7 @@ def _HandlePluginEnd(self, input_line, match, output_stream): self._formatting_handler.HandleEscapedText( input_line, output_stream, - "\n\nUnknown end tag for \n\n".format(tag_without_ns)) + u"\n\nUnknown end tag for \n\n".format(tag_without_ns)) def _HandleVariable(self, input_line, match, output_stream): """Handle a variable. @@ -1250,7 +1245,7 @@ def _HandleVariable(self, input_line, match, output_stream): elif not output and match == "project": if self._project: output = self._project - instructions = ("It has been replaced with static text containing the " + instructions = (u"It has been replaced with static text containing the " "name of the project:\n\t{0}".format(self._project)) else: output = "(TODO: Replace with project name.)" @@ -1260,7 +1255,7 @@ def _HandleVariable(self, input_line, match, output_stream): # Not defined anywhere, just treat as regular text. if not output: # Add surrounding %% back on. - output = "%%{0}%%".format(match) + output = u"%%{0}%%".format(match) self._formatting_handler.HandleEscapedText( input_line, @@ -1269,7 +1264,7 @@ def _HandleVariable(self, input_line, match, output_stream): if instructions: self._warning_method( input_line, - "A variable substitution was performed with %%{0}%%. {1}" + u"A variable substitution was performed with %%{0}%%. {1}" .format(match, instructions)) def _HandleTag(self, input_line, tag, output_stream): diff --git a/wiki-to-md/impl/formatting_handler.py b/wiki_to_md/impl/formatting_handler.py similarity index 93% rename from wiki-to-md/impl/formatting_handler.py rename to wiki_to_md/impl/formatting_handler.py index 75102c8..664fc26 100644 --- a/wiki-to-md/impl/formatting_handler.py +++ b/wiki_to_md/impl/formatting_handler.py @@ -30,7 +30,7 @@ class FormattingHandler(object): # Template for linking to a video. _VIDEO_TEMPLATE = ( - "") @@ -115,7 +115,7 @@ def HandleHeaderOpen(self, input_line, output_stream, header_level): header_level: The header level. """ if self._in_html: - tag = "h{0}".format(header_level) + tag = u"h{0}".format(header_level) self.HandleHtmlOpen(input_line, output_stream, tag, {}, False) else: self._Write("#" * header_level + " ", output_stream) @@ -133,7 +133,7 @@ def HandleHeaderClose( header_level: The header level. """ if self._in_html: - tag = "h{0}".format(header_level) + tag = u"h{0}".format(header_level) self.HandleHtmlClose(input_line, output_stream, tag) else: if self._symmetric_headers: @@ -167,7 +167,7 @@ def HandleCodeBlockOpen(self, input_line, output_stream, specified_language): else: if not specified_language: specified_language = "" - self._Write("```{0}\n".format(specified_language), output_stream) + self._Write(u"```{0}\n".format(specified_language), output_stream) self._in_code_block = True def HandleCodeBlockClose(self, input_line, output_stream): @@ -349,7 +349,7 @@ def HandleSuperscript(self, unused_input_line, output_stream, text): text: The text to output. """ # Markdown currently has no dedicated markup for superscript. - self._Write("{0}".format(text), output_stream) + self._Write(u"{0}".format(text), output_stream) def HandleSubscript(self, unused_input_line, output_stream, text): """Handle the output for subscript text. @@ -360,7 +360,7 @@ def HandleSubscript(self, unused_input_line, output_stream, text): text: The text to output. """ # Markdown currently has no dedicated markup for subscript. - self._Write("{0}".format(text), output_stream) + self._Write(u"{0}".format(text), output_stream) def HandleInlineCode(self, input_line, output_stream, code): """Handle the output for a code block. @@ -391,7 +391,7 @@ def HandleInlineCode(self, input_line, output_stream, code): consecutive_ticks = 0 surrounding_ticks = "`" * (max_consecutive_ticks + 1) - self._Write("{0} {1} {0}".format(surrounding_ticks, code), output_stream) + self._Write(u"{0}{1}{0}".format(surrounding_ticks, code), output_stream) def HandleTableCellBorder(self, input_line, output_stream): """Handle the output for a table cell border. @@ -476,7 +476,7 @@ def HandleTableHeader(self, input_line, output_stream, columns): self.HandleTableCellBorder(input_line, output_stream) # Wiki tables are left-aligned, which takes one character to specify. - self._Write(":{0}".format("-" * (column_width - 1)), output_stream) + self._Write(u":{0}".format("-" * (column_width - 1)), output_stream) self.HandleTableCellBorder(input_line, output_stream) @@ -553,7 +553,7 @@ def HandleLink(self, input_line, output_stream, target, description): # make the link description an inlined image. We do this by setting # the output description to the syntax used to inline an image. if is_image_description: - description = "![]({0})".format(description) + description = u"![]({0})".format(description) elif description: description = self._Escape(description) else: @@ -563,7 +563,7 @@ def HandleLink(self, input_line, output_stream, target, description): # Prefix ! if linking to an image without a text description. prefix = "!" if is_image and is_image_description else "" - output = "{0}[{1}]({2})".format(prefix, description, target) + output = u"{0}[{1}]({2})".format(prefix, description, target) self._Write(output, output_stream) def HandleWiki(self, input_line, output_stream, target, text): @@ -577,7 +577,8 @@ def HandleWiki(self, input_line, output_stream, target, text): """ # A wiki link is just like a regular link, except under the wiki directory. # At this point we make the text equal to the original target if unset. - self.HandleLink(input_line, output_stream, "wiki/" + target, text or target) + # We do however append ".md", assuming the wiki files now have that extension. + self.HandleLink(input_line, output_stream, target + ".md", text or target) def HandleIssue(self, input_line, output_stream, prefix, issue): """Handle the output for an auto-linked issue. @@ -598,10 +599,10 @@ def HandleIssue(self, input_line, output_stream, prefix, issue): input_line, output_stream, migrated_issue_url, - "{0}{1}".format(prefix, migrated_issue)) + u"{0}{1}".format(prefix, migrated_issue)) handled = True - instructions = ("In the output, it has been linked to the migrated issue " + instructions = (u"In the output, it has been linked to the migrated issue " "on GitHub: {0}. Please verify this issue on GitHub " "corresponds to the original issue on Google Code. " .format(migrated_issue)) @@ -619,16 +620,16 @@ def HandleIssue(self, input_line, output_stream, prefix, issue): # If we couldn't handle it in the map, try linking to the old issue. if not handled and self._project: - old_link = ("https://code.google.com/p/{0}/issues/detail?id={1}" + old_link = (u"https://code.google.com/p/{0}/issues/detail?id={1}" .format(self._project, issue)) self.HandleLink( input_line, output_stream, old_link, - "{0}{1}".format(prefix, issue)) + u"{0}{1}".format(prefix, issue)) handled = True - instructions += ("As a placeholder, the text has been modified to " + instructions += (u"As a placeholder, the text has been modified to " "link to the original Google Code issue page:\n\t{0}" .format(old_link)) elif not handled: @@ -638,17 +639,17 @@ def HandleIssue(self, input_line, output_stream, prefix, issue): # Couldn't map it to GitHub nor could we link to the old issue. if not handled: - output = "{0}{1} (on Google Code)".format(prefix, issue) + output = u"{0}{1} (on Google Code)".format(prefix, issue) self._Write(output, output_stream) handled = True - instructions += ("The auto-link has been removed and the text has been " + instructions += (u"The auto-link has been removed and the text has been " "modified from '{0}{1}' to '{2}'." .format(prefix, issue, output)) self._warning_method( input_line, - "Issue {0} was auto-linked. {1}".format(issue, instructions)) + u"Issue {0} was auto-linked. {1}".format(issue, instructions)) def HandleRevision(self, input_line, output_stream, prefix, revision): """Handle the output for an auto-linked issue. @@ -662,22 +663,22 @@ def HandleRevision(self, input_line, output_stream, prefix, revision): # Google Code only auto-linked revision numbers, not hashes, so # revision auto-linking cannot be done for the conversion. if self._project: - old_link = ("https://code.google.com/p/{0}/source/detail?r={1}" + old_link = (u"https://code.google.com/p/{0}/source/detail?r={1}" .format(self._project, revision)) self.HandleLink( input_line, output_stream, old_link, - "{0}{1}".format(prefix, revision)) + u"{0}{1}".format(prefix, revision)) - instructions = ("As a placeholder, the text has been modified to " + instructions = (u"As a placeholder, the text has been modified to " "link to the original Google Code source page:\n\t{0}" .format(old_link)) else: - output = "{0}{1} (on Google Code)".format(prefix, revision) + output = u"{0}{1} (on Google Code)".format(prefix, revision) self._Write(output, output_stream) - instructions = ("Additionally, because no project name was specified " + instructions = (u"Additionally, because no project name was specified " "the revision could not be linked to the original " "Google Code source page. The auto-link has been removed " "and the text has been modified from '{0}{1}' to '{2}'." @@ -685,7 +686,7 @@ def HandleRevision(self, input_line, output_stream, prefix, revision): self._warning_method( input_line, - "Revision {0} was auto-linked. SVN revision numbers are not sensible " + u"Revision {0} was auto-linked. SVN revision numbers are not sensible " "in Git; consider updating this link or removing it altogether. {1}" .format(revision, instructions)) @@ -706,12 +707,12 @@ def HandleHtmlOpen( has_end: True if the tag was self-closed. """ core_params = self._SerializeHtmlParams(params) - core = "{0}{1}".format(html_tag, core_params) + core = u"{0}{1}".format(html_tag, core_params) if has_end: - output = "<{0} />".format(core) + output = u"<{0} />".format(core) else: - output = "<{0}>".format(core) + output = u"<{0}>".format(core) self._in_html += 1 self._Write(output, output_stream) @@ -726,7 +727,7 @@ def HandleHtmlClose(self, unused_input_line, output_stream, html_tag): output_stream: Output Markdown file. html_tag: The HTML tag name. """ - self._Write("".format(html_tag), output_stream) + self._Write(u"".format(html_tag), output_stream) self._in_html -= 1 self._has_written_text = False @@ -738,15 +739,12 @@ def HandleGPlusOpen(self, input_line, output_stream, unused_params): output_stream: Output Markdown file. unused_params: The parameters for the tag. """ - output = "(TODO: Link to Google+ page.)" self._warning_method( input_line, "A Google+ +1 button was embedded on this page, but GitHub does not " "currently support this. Should it become supported in the future, " "see https://developers.google.com/+/web/+1button/ for more " - "information.\nIt has been replaced with:\n\t{0}" - .format(output)) - self._Write(output, output_stream) + "information.\nIt has been removed.") def HandleGPlusClose(self, unused_input_line, unused_output_stream): """Handle the output for closing a +1 button. @@ -844,7 +842,7 @@ def _PrintHtmlWarning(self, input_line, kind): """ self._warning_method( input_line, - "{0} markup was used within HTML tags. Because GitHub does not " + u"{0} markup was used within HTML tags. Because GitHub does not " "support this, the tags have been translated to HTML. Please verify " "that the formatting is correct.".format(kind)) @@ -916,6 +914,10 @@ def _HandleHtmlListClose(self, input_line, output_stream): input_line: Current line number being processed. output_stream: Output Markdown file. """ + # Fix index error if list_tags is empty. + if len(self._list_tags) == 0: + self._warning_method(input_line, "HtmlListClose without list_tags?") + self._list_tags = [ { "indent": 0, "kind": "Bulleted list" } ] top_tag = self._list_tags[-1] kind = top_tag["kind"] self._list_tags.pop() @@ -955,10 +957,10 @@ def _HandleFormatClose(self, input_line, output_stream, kind): self.HandleHtmlClose(input_line, output_stream, tag) else: tag = self._HTML_FORMAT_TAGS[kind]["Markdown"] - self._Write("{0}{1}{0}".format(tag, format_buffer), output_stream) + self._Write(u"{0}{1}{0}".format(tag, format_buffer), output_stream) else: - self._warning_method(input_line, "Re-closed '{0}', ignoring.".format(tag)) + self._warning_method(input_line, u"Re-closed '{0}', ignoring.".format(tag)) def _Indent(self, output_stream, indentation_level): """Output indentation. @@ -987,7 +989,7 @@ def _Escape(self, text): before_match = text[:match.start()] after_match = text[match.end():] escaped_match = match.group(0).replace("<", "<").replace(">", ">") - text = "{0}{1}{2}".format(before_match, escaped_match, after_match) + text = u"{0}{1}{2}".format(before_match, escaped_match, after_match) # In Markdown, if a newline is preceeded by two spaces it breaks the line. # For Wiki text, this is not the case, so we strip such endings off. @@ -1010,7 +1012,7 @@ def _SerializeHtmlParams(self, params): quote = "'" else: quote = "\"" - core_params += " {0}={1}{2}{1}".format(name, quote, value) + core_params += u" {0}={1}{2}{1}".format(name, quote, value) return core_params diff --git a/wiki-to-md/impl/pragma_handler.py b/wiki_to_md/impl/pragma_handler.py similarity index 93% rename from wiki-to-md/impl/pragma_handler.py rename to wiki_to_md/impl/pragma_handler.py index 006bd2d..f2f12c6 100644 --- a/wiki-to-md/impl/pragma_handler.py +++ b/wiki_to_md/impl/pragma_handler.py @@ -44,14 +44,14 @@ def HandlePragma(self, if pragma_type == "summary": self._warning_method( input_line, - "A summary pragma was used for this wiki:\n" + u"A summary pragma was used for this wiki:\n" "\t{0}\n" "Consider moving it to an introductory paragraph." .format(pragma_value)) elif pragma_type == "sidebar": self._warning_method( input_line, - "A sidebar pragma was used for this wiki:\n" + u"A sidebar pragma was used for this wiki:\n" "\t{0}\n" "The Gollum wiki system supports sidebars, and by converting " "{0}.wiki to _Sidebar.md it can be used as a sidebar.\n" @@ -60,7 +60,7 @@ def HandlePragma(self, else: self._warning_method( input_line, - "The following pragma has been ignored:\n" + u"The following pragma has been ignored:\n" "\t#{0} {1}\n" "Consider expressing the same information in a different manner." .format(pragma_type, pragma_value)) diff --git a/wiki-to-md/wiki2gfm.py b/wiki_to_md/wiki2gfm.py similarity index 98% rename from wiki-to-md/wiki2gfm.py rename to wiki_to_md/wiki2gfm.py index 60ab35a..8ee84ec 100755 --- a/wiki-to-md/wiki2gfm.py +++ b/wiki_to_md/wiki2gfm.py @@ -45,7 +45,7 @@ def PrintWarning(input_line, message): input_line: The line number this warning occurred on. message: The warning message. """ - print "Warning (line {0} of input file):\n{1}\n".format(input_line, message) + print u"Warning (line {0} of input file):\n{1}\n".format(input_line, message) def main(args): diff --git a/wiki-to-md/wiki2gfm_test.py b/wiki_to_md/wiki2gfm_test.py similarity index 98% rename from wiki-to-md/wiki2gfm_test.py rename to wiki_to_md/wiki2gfm_test.py index 73eb098..845385d 100644 --- a/wiki-to-md/wiki2gfm_test.py +++ b/wiki_to_md/wiki2gfm_test.py @@ -50,6 +50,9 @@ def assertOutput(self, expected_output): """ self.assertEquals(expected_output, self.output.getvalue()) + def assertNoOutput(self, expected_output): + self.assertNotEqual(expected_output, self.output.getvalue()) + def assertWarning(self, warning_contents, occurrences=1): """Assert that a warning was issued containing the given contents. @@ -390,7 +393,7 @@ def testHandleSubscriptInHtml(self): def testHandleInlineCode(self): self.formatting_handler.HandleInlineCode(1, self.output, "xyz") - self.assertOutput("` xyz `") + self.assertOutput("`xyz`") self.assertNoWarnings() def testHandleInlineCodeInHtml(self): @@ -499,14 +502,14 @@ def testHandleImageLinkWithImageDescriptionInHtml(self): def testHandleWiki(self): self.formatting_handler.HandleWiki(1, self.output, "TestPage", "Test Page") - self.assertOutput("[Test Page](wiki/TestPage)") + self.assertOutput("[Test Page](TestPage.md)") self.assertNoWarnings() def testHandleWikiInHtml(self): self.formatting_handler._in_html = 1 self.formatting_handler.HandleWiki(1, self.output, "TestPage", "Test Page") - self.assertOutput("Test Page") + self.assertOutput("Test Page") self.assertWarning("Link markup was used") def testHandleIssue(self): @@ -707,7 +710,7 @@ def testHandleGPlus(self): self.formatting_handler.HandleGPlusOpen(1, self.output, None) self.formatting_handler.HandleGPlusClose(1, self.output) - self.assertOutput("(TODO: Link to Google+ page.)") + self.assertNoOutput("(TODO: Link to Google+ page.)") self.assertWarning("A Google+ +1 button was embedded on this page") def testHandleGPlusInHtml(self): @@ -715,7 +718,7 @@ def testHandleGPlusInHtml(self): self.formatting_handler.HandleGPlusOpen(1, self.output, None) self.formatting_handler.HandleGPlusClose(1, self.output) - self.assertOutput("(TODO: Link to Google+ page.)") + self.assertNoOutput("(TODO: Link to Google+ page.)") self.assertWarning("A Google+ +1 button was embedded on this page") def testHandleComment(self):