From bdacdc4cb464fe1d43875dc4cdad1ad5a86e39f9 Mon Sep 17 00:00:00 2001 From: nmd2000 Date: Thu, 17 Aug 2023 17:51:58 +0700 Subject: [PATCH 01/10] release 008 --- HISTORY.md | 10 ++++++++++ pyproject.toml | 2 +- src/codetext/codetext_cli.py | 9 ++++++++- src/codetext/parser/java_parser.py | 4 ++++ src/codetext/utils/utils.py | 7 +++++-- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index b84a5c6..7ca2548 100755 --- a/HISTORY.md +++ b/HISTORY.md @@ -85,3 +85,13 @@ Release data: Jul 5, 2023 * Update all class extractor format (using dict instead of list) * Fix missing identifier, parameter in C, C#, Java parser * Implement CLI + +Version 0.0.8 +============= +Release data: Aug 17, 2023 + +* Update format codetext_cli +* Update PythonParser: Handle class definitions with empty argument list class ABC() +* Add Javascript undeclared functions +* Add PHP interface +* Add Ruby actions with block parameters \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8f39ea2..d8bb24d 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "codetext" -version = "0.0.7" +version = "0.0.8" authors = [ { name="Dung Manh Nguyen", email="dungnm.workspace@gmail.com" }, ] diff --git a/src/codetext/codetext_cli.py b/src/codetext/codetext_cli.py index 91f7fd9..36e25ba 100644 --- a/src/codetext/codetext_cli.py +++ b/src/codetext/codetext_cli.py @@ -56,6 +56,7 @@ def parse_file(file_path: str, language: str = None, verbose: bool = False) -> L method_list = parser.get_function_list(_cls) for method in method_list: method_info = parser.get_function_metadata(method) + method_info['code'] = get_node_text(method) cls_method.append(method_info) cls_info["method"] = cls_method @@ -88,7 +89,8 @@ def print_result(res: Dict, file_name: str = "no_name_file"): # ========= Print class & method ========= cls_headers = ["#", "Class", "Arguments"] - cls_method_headers = ["#", "Method name", "Paramters", "Type", "Return type"] + cls_method_headers = ["#", "Method name", "Paramters", + "Type", "Return type", "Throws"] cls_info = [] method_info = {} for cls_idx, _cls in enumerate(res["class"]): @@ -116,6 +118,11 @@ def print_result(res: Dict, file_name: str = "no_name_file"): if i <= 1 and method["return_type"] != "" else "" ) + sublist[5] = ( + method["throws"] + if i <= 1 and "throws" in method.keys() + else "" + ) _method_info.append(sublist) method_info[file_name] = [_cls["identifier"], _method_info] diff --git a/src/codetext/parser/java_parser.py b/src/codetext/parser/java_parser.py index 2046e11..9be34b6 100644 --- a/src/codetext/parser/java_parser.py +++ b/src/codetext/parser/java_parser.py @@ -125,6 +125,10 @@ def get_function_metadata(function_node, blob: str = None) -> Dict[str, str]: metadata['identifier'] = get_node_text(child) elif child.type in return_kinds: metadata['return_type'] = get_node_text(child) + elif child.type == 'throws': + for subchild in child.children: + if 'identifier' in subchild.type: + metadata['throws'] = get_node_text(subchild) elif child.type == 'formal_parameters': param_list = get_node_by_kind(child, ['formal_parameter']) # speed_parameter for param in param_list: diff --git a/src/codetext/utils/utils.py b/src/codetext/utils/utils.py index c3d809f..d330ecb 100644 --- a/src/codetext/utils/utils.py +++ b/src/codetext/utils/utils.py @@ -102,7 +102,10 @@ def parse_code(raw_code: str, language: str='Auto', tree_sitter_path: str=None) parser.set_language(language) if isinstance(raw_code, str): - tree = parser.parse(bytes(raw_code, 'utf8')) - return tree + raw_code = bytes(raw_code, 'utf8') + elif isinstance(raw_code, bytes): + pass else: raise ValueError(f"Expect `str`, got {type(raw_code)}") + tree = parser.parse(raw_code) + return tree From 4c0679891d7abf92d58a918b2f93665be223f583 Mon Sep 17 00:00:00 2001 From: nhtlongcs Date: Sat, 7 Oct 2023 21:36:13 +0000 Subject: [PATCH 02/10] fix class method overwritten --- src/codetext/codetext_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/codetext/codetext_cli.py b/src/codetext/codetext_cli.py index 36e25ba..6ec53c9 100644 --- a/src/codetext/codetext_cli.py +++ b/src/codetext/codetext_cli.py @@ -53,8 +53,8 @@ def parse_file(file_path: str, language: str = None, verbose: bool = False) -> L cls_info["code"] = get_node_text(_cls) cls_method = [] - method_list = parser.get_function_list(_cls) - for method in method_list: + current_class_methods = parser.get_function_list(_cls) + for method in current_class_methods: method_info = parser.get_function_metadata(method) method_info['code'] = get_node_text(method) cls_method.append(method_info) From 99ffe7dc473f2ee9bcb502d02330a410634aa301 Mon Sep 17 00:00:00 2001 From: nhtlongcs Date: Sat, 7 Oct 2023 21:47:01 +0000 Subject: [PATCH 03/10] fix class method overwritten --- src/codetext/codetext_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codetext/codetext_cli.py b/src/codetext/codetext_cli.py index 6ec53c9..43d8824 100644 --- a/src/codetext/codetext_cli.py +++ b/src/codetext/codetext_cli.py @@ -61,7 +61,7 @@ def parse_file(file_path: str, language: str = None, verbose: bool = False) -> L cls_info["method"] = cls_method cls_metadata.append(cls_info) - method_list.extend(method_list) + method_list.extend(current_class_methods) fn_list: List = parser.get_function_list(root_node) for node in fn_list[:]: From 68f3b190029bd6aa9dc6a214fdc69061d7590b81 Mon Sep 17 00:00:00 2001 From: minhna1112 Date: Mon, 20 May 2024 09:41:52 +0700 Subject: [PATCH 04/10] Update .gitignore --- .gitignore | 3 ++- .idea/.gitignore | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .idea/.gitignore diff --git a/.gitignore b/.gitignore index af066ef..6ea0d8c 100755 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ src/codetext.egg-info/* *.pyc *.so *.whl - +.idea +.vscode*.iml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..2840782 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,6 @@ +# Default ignored files +/shelf/ +/workspace.xml +.idea +.vscode +*.iml \ No newline at end of file From 31b5863e2b66720534ed9621464490d6386c1c04 Mon Sep 17 00:00:00 2001 From: minhna1112 Date: Mon, 20 May 2024 09:42:05 +0700 Subject: [PATCH 05/10] Update .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6ea0d8c..bbdf54d 100755 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ src/codetext.egg-info/* *.so *.whl .idea -.vscode*.iml +.vscode +*.iml From 3876ce2e42d34d2661261f219379da7562c4a52d Mon Sep 17 00:00:00 2001 From: minhna1112 Date: Mon, 20 May 2024 09:42:50 +0700 Subject: [PATCH 06/10] skip parser build (Attempt #1) --- pyproject.toml | 5 +++-- requirements.txt | 1 + src/codetext/utils/utils.py | 22 ++++++++++++++-------- tests/setup.py | 8 ++++++-- tests/test_utils/test_utils.py | 2 -- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d8bb24d..b6a68f6 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "codetext" -version = "0.0.8" +version = "0.0.9" authors = [ { name="Dung Manh Nguyen", email="dungnm.workspace@gmail.com" }, ] @@ -21,7 +21,8 @@ dependencies = [ "Levenshtein>=0.20", "langdetect>=1.0.0", "bs4>=0.0.1", - "tabulate>=0.9.0" + "tabulate>=0.9.0", + "tree_sitter_languages>=1.10.0" ] [project.urls] diff --git a/requirements.txt b/requirements.txt index d438040..9eb91b2 100755 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ tabulate Levenshtein langdetect bs4 +tree-sitter-languages \ No newline at end of file diff --git a/src/codetext/utils/utils.py b/src/codetext/utils/utils.py index d330ecb..16c94f8 100644 --- a/src/codetext/utils/utils.py +++ b/src/codetext/utils/utils.py @@ -92,15 +92,21 @@ def parse_code(raw_code: str, language: str='Auto', tree_sitter_path: str=None) calling_script_path = Path(inspect.getframeinfo(sys._getframe(1)).filename) load_path = str(calling_script_path.parent) - ts_lang_path = os.path.join(load_path, 'tree-sitter', f'{language}.so') - if not os.path.exists(ts_lang_path): - logger.warning(f"Not found `{language}.so` in `{load_path}/tree-sitter/`, attemp to build language") - build_language(language, load_path) - + # Get parser from languages parser = Parser() - language = Language(load_path + f"/tree-sitter/{language}.so", language) - parser.set_language(language) - + try: + from tree_sitter_languages import get_language, get_parser + parser = get_parser(get_language(language)) + except ImportError: + # Work-around when pre-built binaries wheels for tree-sitter-languages are not available + logger.warning(f"Troubled importing 'tree-sitter-languages', attemp to look for pre-built binaries in the workspace") + ts_lang_path = os.path.join(load_path, 'tree-sitter', f'{language}.so') + if not os.path.exists(ts_lang_path): + logger.warning(f"Not found `{language}.so` in `{load_path}/tree-sitter/`, attemp to build language") + build_language(language, load_path) + language = Language(load_path + f"/tree-sitter/{language}.so", language) + parser.set_language(language) + if isinstance(raw_code, str): raw_code = bytes(raw_code, 'utf8') elif isinstance(raw_code, bytes): diff --git a/tests/setup.py b/tests/setup.py index 4a7f6aa..d9516a6 100755 --- a/tests/setup.py +++ b/tests/setup.py @@ -1,8 +1,12 @@ from ..src.codetext.utils import build_language - +from tree_sitter_languages import get_language, get_parser if __name__ == '__main__': lang_list = ['python', 'cpp', 'java', 'c-sharp', 'ruby', 'rust', 'javascript', 'php', 'go'] for lang in lang_list: - build_language(lang) + # build_language(lang) + try: + get_parser(get_language(lang)) + except: + build_language(lang) diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index d4a4ba4..af7288c 100755 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -14,8 +14,6 @@ def test_parse_code(self): def sum_2_num(a, b): return a + b """ - - build_language(language='python') parse_code(sample, 'python') From ced3bf6e83d0975a2cc31a5d0f0fc13cc0c62ead Mon Sep 17 00:00:00 2001 From: minhna1112 Date: Mon, 1 Jul 2024 10:46:45 +0700 Subject: [PATCH 07/10] # Please enter the commit message for your changes. Lines starting # with '#' will be ignored, and an empty message aborts the commit. # # On branch dev/v0.0.9 # Changes to be committed: # modified: requirements.txt # Update `requirements` - Current version of `codetext` can only work with `tree-sitter==0.20.4` - Replace current language builders with `tree_sitter_languages==1.10.2` --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d438040..4bc4c06 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # for preprocessing -tree-sitter +tree-sitter==0.20.4 tabulate Levenshtein langdetect bs4 +tree_sitter_languages==1.10.2 From 44fafa1694a11a5bc36ade7db7bc242f6b78a4c7 Mon Sep 17 00:00:00 2001 From: minhna1112 Date: Mon, 1 Jul 2024 11:09:56 +0700 Subject: [PATCH 08/10] Skip language build, use pre-built `get_language` --- src/codetext/utils/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/codetext/utils/utils.py b/src/codetext/utils/utils.py index 16c94f8..98d57e5 100644 --- a/src/codetext/utils/utils.py +++ b/src/codetext/utils/utils.py @@ -96,7 +96,7 @@ def parse_code(raw_code: str, language: str='Auto', tree_sitter_path: str=None) parser = Parser() try: from tree_sitter_languages import get_language, get_parser - parser = get_parser(get_language(language)) + language = get_language(language) except ImportError: # Work-around when pre-built binaries wheels for tree-sitter-languages are not available logger.warning(f"Troubled importing 'tree-sitter-languages', attemp to look for pre-built binaries in the workspace") @@ -104,9 +104,9 @@ def parse_code(raw_code: str, language: str='Auto', tree_sitter_path: str=None) if not os.path.exists(ts_lang_path): logger.warning(f"Not found `{language}.so` in `{load_path}/tree-sitter/`, attemp to build language") build_language(language, load_path) - language = Language(load_path + f"/tree-sitter/{language}.so", language) - parser.set_language(language) - + language = Language(load_path + f"/tree-sitter/{language}.so", language) + parser.set_language(language) + if isinstance(raw_code, str): raw_code = bytes(raw_code, 'utf8') elif isinstance(raw_code, bytes): From 3daaa8ac83d468888d7467bd04b9a6ade7fb1135 Mon Sep 17 00:00:00 2001 From: minhna1112 Date: Mon, 1 Jul 2024 11:11:23 +0700 Subject: [PATCH 09/10] Fix indentation --- src/codetext/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codetext/utils/utils.py b/src/codetext/utils/utils.py index 98d57e5..5975897 100644 --- a/src/codetext/utils/utils.py +++ b/src/codetext/utils/utils.py @@ -104,7 +104,7 @@ def parse_code(raw_code: str, language: str='Auto', tree_sitter_path: str=None) if not os.path.exists(ts_lang_path): logger.warning(f"Not found `{language}.so` in `{load_path}/tree-sitter/`, attemp to build language") build_language(language, load_path) - language = Language(load_path + f"/tree-sitter/{language}.so", language) + language = Language(load_path + f"/tree-sitter/{language}.so", language) parser.set_language(language) if isinstance(raw_code, str): From d2a7365f7f944650e84d9fdb6b6794d6c5ea620b Mon Sep 17 00:00:00 2001 From: nmd2k Date: Mon, 1 Jul 2024 08:41:14 +0000 Subject: [PATCH 10/10] update history --- HISTORY.md | 133 +++++++++++++++++++++++++------------------------ pyproject.toml | 5 +- 2 files changed, 71 insertions(+), 67 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 7ca2548..a8e5025 100755 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,36 +2,55 @@ Releases ======== -Version 0.0.1 +Version 0.0.9 ============= -Release date: Nov 9, 2022 +Release date: Jul 1, 2024 +* Skip building language binaries from source -* Language parser for Java, Python, JavaScript, PHP, Golang, Ruby, C++, C#, C - * get_docstring - * get_class_list, get_function_list - * get_class_metadata, get_function_metadata -* Clean docstring function -* Data preprocessing source code -* Tree-sitter utils: build_language, parse_code +Version 0.0.8 +============= +Release date: Aug 17, 2023 -Version 0.0.2 +* Update format codetext_cli +* Update PythonParser: Handle class definitions with empty argument list class ABC() +* Add Javascript undeclared functions +* Add PHP interface +* Add Ruby actions with block parameters + +Version 0.0.7 ============= -Release date: Nov 25, 2022 +Release date: Jul 5, 2023 -* Language parser for Rust - * get_docstring - * get_class_list, get_function_list - * get_class_metadata, get_function_metadata -* Processing utils: - * extract_docstring - * extract_node - * get_line_definitions - * get_node_definitions - * process_raw_node -* Postprocessing: - * Merge file (from batches) - * Split into train/test/valid (by #sample category) - * Deduplicate sample +* Update all class extractor format (using dict instead of list) +* Fix missing identifier, parameter in C, C#, Java parser +* Implement CLI + +Version 0.0.6 +============= +Release date: Jan 9, 2023 + +* Add tree sitter utils (in codetext.parser) +* Replace all `match_from_span` to `get_node_text` +* Replace all `traverse_type` to `get_node_by_kind` +* Fix `CppParser.get_function_metadata` missing `param_type` and `param_identifier` +* Update return metadata from all parser + +Version 0.0.5 +============= +Release date: Dec 12, 2022 + +* Fix package import path +* Adding auto build workflow +* Seperate codetext parser with processing source code +* Fix `remove_comment_delimiter` remove leading whitespace +* Update unittest for parser and utilites + +Version 0.0.4 +============= +Release date: Dec 2, 2022 + +* Fix main package root path +* Loosen `docstring_parser` dependency Version 0.0.3 ============= @@ -51,47 +70,33 @@ Release date: Dec 2, 2022 * check_contain_many_uppercase_word * check_contain_many_long_word -Version 0.0.4 -============= -Release date: Dec 2, 2022 - -* Fix main package root path -* Loosen `docstring_parser` dependency - -Version 0.0.5 -============= -Release data: Dec 12, 2022 - -* Fix package import path -* Adding auto build workflow -* Seperate codetext parser with processing source code -* Fix `remove_comment_delimiter` remove leading whitespace -* Update unittest for parser and utilites - -Version 0.0.6 -============= -Release data: Jan 9, 2023 - -* Add tree sitter utils (in codetext.parser) -* Replace all `match_from_span` to `get_node_text` -* Replace all `traverse_type` to `get_node_by_kind` -* Fix `CppParser.get_function_metadata` missing `param_type` and `param_identifier` -* Update return metadata from all parser - -Version 0.0.7 +Version 0.0.2 ============= -Release data: Jul 5, 2023 +Release date: Nov 25, 2022 -* Update all class extractor format (using dict instead of list) -* Fix missing identifier, parameter in C, C#, Java parser -* Implement CLI +* Language parser for Rust + * get_docstring + * get_class_list, get_function_list + * get_class_metadata, get_function_metadata +* Processing utils: + * extract_docstring + * extract_node + * get_line_definitions + * get_node_definitions + * process_raw_node +* Postprocessing: + * Merge file (from batches) + * Split into train/test/valid (by #sample category) + * Deduplicate sample -Version 0.0.8 +Version 0.0.1 ============= -Release data: Aug 17, 2023 +Release date: Nov 9, 2022 -* Update format codetext_cli -* Update PythonParser: Handle class definitions with empty argument list class ABC() -* Add Javascript undeclared functions -* Add PHP interface -* Add Ruby actions with block parameters \ No newline at end of file +* Language parser for Java, Python, JavaScript, PHP, Golang, Ruby, C++, C#, C + * get_docstring + * get_class_list, get_function_list + * get_class_metadata, get_function_metadata +* Clean docstring function +* Data preprocessing source code +* Tree-sitter utils: build_language, parse_code diff --git a/pyproject.toml b/pyproject.toml index 9db3f28..4baf007 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,12 +17,11 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "tree-sitter==0.20.4", + "tree-sitter>=0.20", "Levenshtein>=0.20", "langdetect>=1.0.0", "bs4>=0.0.1", - "tabulate>=0.9.0", - "tree_sitter_languages>=1.10.0" + "tabulate>=0.9.0" ] [project.urls]