--- .../comment-filter/.circleci/config.yml | 9 - .../src/modules/comment-filter/.coveragerc | 2 - .../src/src/modules/comment-filter/.gitignore | 104 ------- .../modules/comment-filter/CODE-OF-CONDUCT.md | 73 ----- .../modules/comment-filter/CONTRIBUTING.md | 87 ------ .../src/src/modules/comment-filter/Dockerfile | 9 - .../src/src/modules/comment-filter/LICENSE | 4 +- .../src/modules/comment-filter/MANIFEST.in | 7 - .../src/modules/comment-filter/bin/comments | 25 -- .../comment-filter/bin/testdata/hello.c | 8 - .../comment-filter/comment_filter/_version.py | 0 .../comment-filter/comment_filter/language.py | 14 +- .../comment-filter/comment_filter/rfc.py | 62 +++-- .../comment-filter/comment_filter/rfc_test.py | 258 ------------------ 14 files changed, 42 insertions(+), 620 deletions(-) delete mode 100644 formal/promela/src/src/modules/comment-filter/.circleci/config.yml delete mode 100644 formal/promela/src/src/modules/comment-filter/.coveragerc delete mode 100644 formal/promela/src/src/modules/comment-filter/.gitignore delete mode 100644 formal/promela/src/src/modules/comment-filter/CODE-OF-CONDUCT.md delete mode 100644 formal/promela/src/src/modules/comment-filter/CONTRIBUTING.md delete mode 100644 formal/promela/src/src/modules/comment-filter/Dockerfile delete mode 100644 formal/promela/src/src/modules/comment-filter/MANIFEST.in delete mode 100755 formal/promela/src/src/modules/comment-filter/bin/comments delete mode 100644 formal/promela/src/src/modules/comment-filter/bin/testdata/hello.c delete mode 100644 formal/promela/src/src/modules/comment-filter/comment_filter/_version.py delete mode 100644 formal/promela/src/src/modules/comment-filter/comment_filter/rfc_test.py
diff --git a/formal/promela/src/src/modules/comment-filter/.circleci/config.yml b/formal/promela/src/src/modules/comment-filter/.circleci/config.yml deleted file mode 100644 index dc96c5c2..00000000 --- a/formal/promela/src/src/modules/comment-filter/.circleci/config.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: themattrix/tox - steps: - - checkout - - run: - command: tox diff --git a/formal/promela/src/src/modules/comment-filter/.coveragerc b/formal/promela/src/src/modules/comment-filter/.coveragerc deleted file mode 100644 index 9b6154a0..00000000 --- a/formal/promela/src/src/modules/comment-filter/.coveragerc +++ /dev/null @@ -1,2 +0,0 @@ -[run] -omit = *_test.py diff --git a/formal/promela/src/src/modules/comment-filter/.gitignore b/formal/promela/src/src/modules/comment-filter/.gitignore deleted file mode 100644 index af2f5375..00000000 --- a/formal/promela/src/src/modules/comment-filter/.gitignore +++ /dev/null @@ -1,104 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -.static_storage/ -.media/ -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ diff --git a/formal/promela/src/src/modules/comment-filter/CODE-OF-CONDUCT.md b/formal/promela/src/src/modules/comment-filter/CODE-OF-CONDUCT.md deleted file mode 100644 index 7ef343f5..00000000 --- a/formal/promela/src/src/modules/comment-filter/CODE-OF-CONDUCT.md +++ /dev/null @@ -1,73 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of experience, -nationality, personal appearance, race, religion, or sexual identity and -orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team. All complaints will be reviewed -and investigated and will result in a response that is deemed necessary and -appropriate to the circumstances. The project team is obligated to maintain -confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html - -[homepage]: https://www.contributor-covenant.org diff --git a/formal/promela/src/src/modules/comment-filter/CONTRIBUTING.md b/formal/promela/src/src/modules/comment-filter/CONTRIBUTING.md deleted file mode 100644 index 02f6b9ff..00000000 --- a/formal/promela/src/src/modules/comment-filter/CONTRIBUTING.md +++ /dev/null @@ -1,87 +0,0 @@ -## Contributing to Comment Filter - -Hi there! -We’re thrilled that you’d like to contribute to this project. -Your help is essential for keeping this project great and for making it better. - -## Branching Strategy - -In general, contributors should develop on branches based off of `master` and pull requests should be made against `master`. - -## Submitting a pull request - -1. Please read our [code of conduct](code-of-conduct.md] and [license](LICENSE.txt). -1. [Fork](https://github.com/codeauroraforum/comment-filter/fork) and clone the repository. -1. Create a new branch based on `master`: `git checkout -b <my-branch-name> master`. -1. Make your changes, add tests, and make sure the tests still pass. -1. Push to your fork and [submit a pull request](https://github.com/codeauroraforum/comment-filter/compare) from your branch to `master`. -1. Pat yourself on the back and wait for your pull request to be reviewed. - -Here are a few things you can do that will increase the likelihood of your pull request to be accepted: - -- Follow the existing style where possible. We try and adhere to [pep8](https://www.python.org/dev/peps/pep-0008/). -- Write tests. -- Keep your change as focused as possible. - If you want to make multiple independent changes, please consider submitting them as separate pull requests. -- Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). - - -## Developer Certification of Origin (DCO) -Comment Filter requires the Developer Certificate of Origin (DCO) process to be followed. - -The DCO is an attestation attached to every contribution made by every developer. In the commit message of the contribution, the developer simply adds a Signed-off-by statement and thereby agrees to the DCO, which you can find below or at http://developercertificate.org/. - -Comment Filter does not merge any pull requests made until each commit has been signed for the DCO. - -``` -Developer Certificate of Origin -Version 1.1 - -Copyright (C) 2004, 2006 The Linux Foundation and its contributors. -1 Letterman Drive -Suite D4700 -San Francisco, CA, 94129 - -Everyone is permitted to copy and distribute verbatim copies of this -license document, but changing it is not allowed. - - -Developer's Certificate of Origin 1.1 - -By making a contribution to this project, I certify that: - -(a) The contribution was created in whole or in part by me and I - have the right to submit it under the open source license - indicated in the file; or - -(b) The contribution is based upon previous work that, to the best - of my knowledge, is covered under an appropriate open source - license and I have the right under that license to submit that - work with modifications, whether created in whole or in part - by me, under the same open source license (unless I am - permitted to submit under a different license), as indicated - in the file; or - -(c) The contribution was provided directly to me by some other - person who certified (a), (b) or (c) and I have not modified - it. - -(d) I understand and agree that this project and the contribution - are public and that a record of the contribution (including all - personal information I submit with it, including my sign-off) is - maintained indefinitely and may be redistributed consistent with - this project or the open source license(s) involved. - ``` - -### DCO commit example - -After committing your changes with `git commit -s`, your message should look something like: -``` -commit 442deae270bf585052be012b064ed92299e221c4 -Author: Random Developer <ran...@developer.org> -Date: Sat Oct 21 08:33:15 2017 -0700 - - My example commit message - - Signed-off-by: Random Developer <ran...@developer.org> -``` diff --git a/formal/promela/src/src/modules/comment-filter/Dockerfile b/formal/promela/src/src/modules/comment-filter/Dockerfile deleted file mode 100644 index 2fdb19e9..00000000 --- a/formal/promela/src/src/modules/comment-filter/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM ubuntu:14.04 - -MAINTAINER Craig Northway - -RUN apt-get update && apt-get install -y python python3 python-pip git && pip install -U setuptools==25.2.0 tox - -COPY . /src - -RUN tox -c /src/tox.ini diff --git a/formal/promela/src/src/modules/comment-filter/LICENSE b/formal/promela/src/src/modules/comment-filter/LICENSE index b8bf6d0f..158c0dc5 100644 --- a/formal/promela/src/src/modules/comment-filter/LICENSE +++ b/formal/promela/src/src/modules/comment-filter/LICENSE @@ -1,4 +1,6 @@ -Copyright (c) 2017, The Linux Foundation. All rights reserved. +Copyright (c) 2019-2020 Trinity College Dublin, Ireland +Copyright (c) 2017, The Linux Foundation. +All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/formal/promela/src/src/modules/comment-filter/MANIFEST.in b/formal/promela/src/src/modules/comment-filter/MANIFEST.in deleted file mode 100644 index f800456f..00000000 --- a/formal/promela/src/src/modules/comment-filter/MANIFEST.in +++ /dev/null @@ -1,7 +0,0 @@ -include *.md -include .coveragerc -include AUTHORS -include LICENSE -include tox.ini -exclude Dockerfile -recursive-include bin *.c diff --git a/formal/promela/src/src/modules/comment-filter/bin/comments b/formal/promela/src/src/modules/comment-filter/bin/comments deleted file mode 100755 index 0b9c7d5a..00000000 --- a/formal/promela/src/src/modules/comment-filter/bin/comments +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python - -import sys -import os -import fileinput -from comment_filter import language -import comment_filter -import argparse -from comment_filter import _version - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--onlycode', help='filter out comments', action='store_true') - parser.add_argument('--notokens', help='filter out comment tokens', action='store_true') - parser.add_argument('--version', action='version', version=_version.__version__) - parser.add_argument('path', help='path to file to parse') - args = parser.parse_args() - - _, ext = os.path.splitext(args.path) - lang = language.extension_to_lang_map.get(ext, language.c) - input_stream = fileinput.input(args.path) - keep_tokens = not args.notokens - for line in comment_filter.parse_file(lang, input_stream, code_only=args.onlycode, keep_tokens=keep_tokens): - sys.stdout.write(line) diff --git a/formal/promela/src/src/modules/comment-filter/bin/testdata/hello.c b/formal/promela/src/src/modules/comment-filter/bin/testdata/hello.c deleted file mode 100644 index 0baf6932..00000000 --- a/formal/promela/src/src/modules/comment-filter/bin/testdata/hello.c +++ /dev/null @@ -1,8 +0,0 @@ -/* multi-line - comment */ -// single-line comment - -int main() { - return 0; -} - diff --git a/formal/promela/src/src/modules/comment-filter/comment_filter/_version.py b/formal/promela/src/src/modules/comment-filter/comment_filter/_version.py deleted file mode 100644 index e69de29b..00000000 diff --git a/formal/promela/src/src/modules/comment-filter/comment_filter/language.py b/formal/promela/src/src/modules/comment-filter/comment_filter/language.py index a934810e..ce96f5df 100644 --- a/formal/promela/src/src/modules/comment-filter/comment_filter/language.py +++ b/formal/promela/src/src/modules/comment-filter/comment_filter/language.py @@ -7,37 +7,37 @@ class Lang: self.string_literal2_start = "'" c = Lang( - line_comment='//', + line_comment=['//'], comment_bookends=[('/*', '*/'), (';;', ';;')], nested_comments=False) haskell = Lang( - line_comment='--', + line_comment=['--'], comment_bookends=[('{-', '-}')], nested_comments=True) python = Lang( - line_comment='#', + line_comment=['#'], comment_bookends=[('"""', '"""'), ("'''", "'''")], nested_comments=False) ruby = Lang( - line_comment='#', + line_comment=['#'], comment_bookends=[("=begin", "=end")], nested_comments=False) lua = Lang( - line_comment='--', + line_comment=['--'], comment_bookends=[("--[[", "--]]")], nested_comments=False) perl = Lang( - line_comment='#', + line_comment=['#'], comment_bookends=[("=pod", "=cut")], nested_comments=False) java = Lang( - line_comment='//', + line_comment=['//'], comment_bookends=[('/*', '*/')], nested_comments=True) diff --git a/formal/promela/src/src/modules/comment-filter/comment_filter/rfc.py b/formal/promela/src/src/modules/comment-filter/comment_filter/rfc.py index 86b7deaa..bf9cf04e 100644 --- a/formal/promela/src/src/modules/comment-filter/comment_filter/rfc.py +++ b/formal/promela/src/src/modules/comment-filter/comment_filter/rfc.py @@ -73,6 +73,7 @@ def parse_line(lang, state, code_only=False, keep_tokens=True): # If currently within a string literal or multi-line comment, first # complete parsing that declaration. Store the result in 'rest_of_decl'. rest_of_decl = '' + rest_of_decl0 = '' if state.in_literal: # Parsing a string literal. cnts, state = finish_string_literal(state.in_literal, state) @@ -83,17 +84,19 @@ def parse_line(lang, state, code_only=False, keep_tokens=True): elif state.multi_end_stack: # If there is state, we assume it is because we have parsed # the start of a multiline comment, but haven't found the end. - cmt, state = finish_multiline_comment(lang, state, keep_tokens) + cmt, cmt0, state = finish_multiline_comment(lang, state, keep_tokens) if code_only: rest_of_decl = clear_line(cmt) + rest_of_decl0 = clear_line(cmt0) else: rest_of_decl = cmt + rest_of_decl0 = cmt0 if state.in_literal or state.multi_end_stack: - return rest_of_decl, state + return [rest_of_decl + rest_of_decl0], state decls, state = parse_declarations(lang, state, code_only, keep_tokens) - return rest_of_decl + decls, state + return [rest_of_decl, rest_of_decl0] + decls, state def parse_declarations(lang, state, code_only=False, keep_tokens=True): @@ -120,24 +123,24 @@ def parse_declarations(lang, state, code_only=False, keep_tokens=True): """ code, state = parse_code(lang, state) comment, state = parse_line_comment(lang, state, keep_tokens) - comment2, state = parse_multiline_comment(lang, state, keep_tokens) + comment2, comment2_0, state = parse_multiline_comment(lang, state, keep_tokens) - if comment or comment2: - line = state.line + if comment or comment2 or comment2_0: + line = [state.line] if not state.multi_end_stack: # Continue looking for declarations. line, state = parse_declarations(lang, state, code_only, keep_tokens) if code_only: - line = code + clear_line(comment) + clear_line(comment2) + line + line = [code, clear_line(comment), clear_line(comment2), clear_line(comment2_0)] + line else: - line = clear_line(code) + comment + comment2 + line + line = [clear_line(code), comment, comment2, comment2_0] + line return line, state else: state.line = '' if code_only: - return code, state + return [code], state else: - return clear_line(code), state + return [clear_line(code)], state def parse_code(lang, state): @@ -157,15 +160,14 @@ def parse_code(lang, state): while True: line = state.line multi_start_tokens = [start for start, end in lang.comment_bookends] - tokens = multi_start_tokens + [ - lang.line_comment, + tokens = multi_start_tokens + lang.line_comment + [ lang.string_literal_start, lang.string_literal2_start] i = index_of_first_found(line, tokens) if i != -1: state.line = line[i:] code += line[:i] - if line.startswith(lang.line_comment, i) or \ + if [ () for cmt in lang.line_comment if line.startswith(cmt, i) ] or \ index_of_first_found(line, multi_start_tokens) == i: return code, state elif line.startswith(lang.string_literal_start, i): @@ -271,15 +273,14 @@ def parse_line_comment(lang, state, keep_tokens=True): (string, State) """ line = state.line - line_comment = lang.line_comment - if line.startswith(line_comment): - state.line = '' - i = len(line_comment) - if not keep_tokens: - line_comment = ' ' * i - return line_comment + line[i:], state - else: - return '', state + for line_comment in lang.line_comment: + if line.startswith(line_comment): + state.line = '' + i = len(line_comment) + if not keep_tokens: + line_comment = ' ' * i + return line_comment + line[i:], state + return '', state def parse_multiline_comment(lang, state, keep_tokens=True): @@ -304,11 +305,11 @@ def parse_multiline_comment(lang, state, keep_tokens=True): if line.startswith(multi_start): state.multi_end_stack.append(multi_end) state.line = line[len(multi_start):] - cnts, state = finish_multiline_comment(lang, state, keep_tokens) + cnts, cnts0, state = finish_multiline_comment(lang, state, keep_tokens) if not keep_tokens: multi_start = ' ' * len(multi_start) - return multi_start + cnts, state - return '', state + return multi_start + cnts, cnts0, state + return '', '', state def finish_multiline_comment(lang, state, keep_tokens=True): @@ -332,9 +333,10 @@ def finish_multiline_comment(lang, state, keep_tokens=True): # Handle language supports nested comments. if lang.nested_comments: - cmt, state = parse_multiline_comment(lang, state, keep_tokens) + cmt, cmt0, state = parse_multiline_comment(lang, state, keep_tokens) else: cmt = '' + cmt0 = '' line = state.line if line: @@ -344,12 +346,12 @@ def finish_multiline_comment(lang, state, keep_tokens=True): state.line = line[i:] if not keep_tokens: multi_end = ' ' * len(multi_end) - return cnts + cmt + multi_end, state + return cnts + cmt + cmt0, multi_end, state else: - more_cnts, state = finish_multiline_comment(lang, state, keep_tokens) - return cnts + cmt + more_cnts, state + more_cnts, more_cnts0, state = finish_multiline_comment(lang, state, keep_tokens) + return cnts + cmt + cmt0 + more_cnts, more_cnts0, state else: - return cnts + cmt, state + return cnts + cmt + cmt0, '', state def parse_multiline_contents(lang, state): diff --git a/formal/promela/src/src/modules/comment-filter/comment_filter/rfc_test.py b/formal/promela/src/src/modules/comment-filter/comment_filter/rfc_test.py deleted file mode 100644 index dd626a68..00000000 --- a/formal/promela/src/src/modules/comment-filter/comment_filter/rfc_test.py +++ /dev/null @@ -1,258 +0,0 @@ -from . import rfc -from . import language -from functools import reduce, wraps -from sys import getrecursionlimit, setrecursionlimit - -try: - from cStringIO import StringIO -except: - from io import StringIO - - -# Same as rfc.parse_line(), but ensure it always returns a string with the -# same length as the input string. -def safe_parse_line(lang, state, **kwargs): - old_line = state.line - new_line, new_state = rfc.parse_line(lang, state, **kwargs) - assert len(new_line) == len(old_line) - return new_line, new_state - - -def make_state(line='', multi_end_stack=None, in_literal=None): - return rfc.State(line, multi_end_stack, in_literal) - - -def test_state(): - """ - Verify constructor doesn't return a global default value. - """ - rfc.State().multi_end_stack.append('doh!') - assert(len(rfc.State().multi_end_stack) == 0) - - -def c_line(s): - """ - Given a string, return only the C comments, and in the column same position. - """ - line, state = safe_parse_line(language.c, make_state(s)) - assert state.multi_end_stack == [] - return line - - -def c_code(s): - """ - Given a string, return only the C code, and in the column same position. - """ - line, state = safe_parse_line(language.c, make_state(s), code_only=True) - assert state.multi_end_stack == [] - return line - - -def py_line(s): - """ - Given a string, return only the Python comments, and in the column same position. - """ - line, state = safe_parse_line(language.python, make_state(s)) - assert state.multi_end_stack == [] - return line - - -def java_line(s): - """ - Given a string, return only the Java comments, and in the column same position. - Unlike c_line, java_line supports nested comments. - """ - line, state = safe_parse_line(language.java, make_state(s)) - assert state.multi_end_stack == [] - return line - - -def parse_code(lang, line): - return rfc.parse_code(lang, make_state(line)) - -def test_parse_code(): - c = language.c - assert parse_code(c, '') == ('', make_state('')) - assert parse_code(c, '\n') == ('\n', make_state('')) - assert parse_code(c, 'foo // bar') == ('foo ', make_state('// bar')) - assert parse_code(c, 'foo /* bar') == ('foo ', make_state('/* bar')) # Ensure '*' is escapted. - assert parse_code(c, '/**/ foo') == ('', make_state('/**/ foo')) # Only return code /before/ comments. - - # Multi-line Python strings are treated as comments. - assert parse_code(language.python, '""" bar """') == ('', make_state('""" bar """')) - - -def line_comment(lang, line, keep_tokens=True): - cmt, state = rfc.parse_line_comment(lang, make_state(line), keep_tokens) - return (cmt, state.line) - - -def test_parse_line_comment(): - c = language.c - assert line_comment(c, '') == ('', '') - assert line_comment(c, '\n') == ('', '\n') - assert line_comment(c, '//\n') == ('//\n', '') - assert line_comment(c, '//\n', False) == (' \n', '') - - -def test_index_of_first_found(): - assert rfc.index_of_first_found('', []) == 0 # Nothing to find. - assert rfc.index_of_first_found('', ['a']) == -1 # Nothing found. - assert rfc.index_of_first_found('ab', ['b']) == 1 # 'b' found. - assert rfc.index_of_first_found('abc', ['b', 'c']) == 1 # 'b' found first. - assert rfc.index_of_first_found('acb', ['b', 'c']) == 1 # 'c' found first. - - -def test_clear_line(): - assert rfc.clear_line('') == '' - assert rfc.clear_line('abc') == ' ' - assert rfc.clear_line('abc\n') == ' \n' # Preserve newline. - assert rfc.clear_line('abc\r\n') == ' \r\n' # Preserve multibyte newline. - - -def test_get_linesep(): - assert rfc.get_linesep('') == '' - assert rfc.get_linesep('foo\n') == '\n' - assert rfc.get_linesep('foo\r\n') == '\r\n' - assert rfc.get_linesep('foo\r\nbar\n') == '\n' - - -def multiline_comment(lang, line, keep_tokens=True): - return rfc.parse_multiline_comment(lang, make_state(line), keep_tokens) - - -def test_parse_multiline_comment(): - c = language.c - assert multiline_comment(c, '/**/\n') == ('/**/', make_state('\n')) - assert multiline_comment(c, '/**/\n', False) == (' ', make_state('\n')) - - -def declarations(lang, line, keep_tokens=True): - return rfc.parse_declarations(lang, make_state(line), keep_tokens=keep_tokens) - - -def test_parse_declarations(): - c = language.c - assert declarations(c, '\n') == ('\n', make_state()) - assert declarations(c, '/**/\n') == ('/**/\n', make_state()) - - -def test_parse_line(): - assert c_line('') == '' - assert c_line('no comments') == ' ' - assert c_line('/**/') == '/**/' - assert c_line('/* a */') == '/* a */' - assert c_line('// a') == '// a' - assert py_line('""" a """') == '""" a """' - assert py_line("''' a '''") == "''' a '''" - - # Preserve newline - assert c_line('/* a */\n') == '/* a */\n' - - # Ensure column position is not modified. - assert c_line('abc /* a */') == ' /* a */' - assert c_line('abc // a') == ' // a' - - # Test comments in comments - assert c_line('// /*abc*/') == '// /*abc*/' - assert c_line('/* a */ // a') == '/* a */ // a' - assert java_line('/* /**/ */') == '/* /**/ */' - assert java_line('/*/**/*/') == '/*/**/*/' - assert c_line('/*/**/*/') == '/*/**/ ' - assert c_line('/* // */') == '/* // */' - assert py_line('"""# a"""') == '"""# a"""' - - # Test strings with strings - assert c_line('"\\\"foo\\\""') == ' ' - assert c_line('"foo') == ' ' - - # Test string literals with comments - assert c_line('"/*"') == ' ' - assert c_line("'/*'") == ' ' - - # Test c_code - assert c_code('') == '' - assert c_code('no comments') == 'no comments' - assert c_code('/**/') == ' ' - assert c_code('/* a */\n') == ' \n' - assert c_code('/* a */ abc') == ' abc' - assert c_code('abc /**/') == 'abc ' - - -def string_literal(quote, line): - lit, state = rfc.parse_string_literal(quote, make_state(line)) - return (lit, state.line) - - -def test_parse_string_literal(): - assert string_literal('"', 'abc') == ('', 'abc') - assert string_literal('"', '"a"') == ('"a"', '') - assert string_literal("'", "'a'") == ("'a'", '') - assert string_literal("'", "'a'b") == ("'a'", 'b') - - # String without an end quote. - assert string_literal('"', '"a') == ('"a', '') - - -def parse_line(lang, line, multi_end_stack=None, code_only=False): - return safe_parse_line(lang, make_state(line, multi_end_stack), code_only=code_only) - - -def test_incomplete_multiline(): - c = language.c - assert parse_line(c, '/* a\n') == ('/* a\n', make_state('', ['*/'])) - - -def test_incomplete_string_literal(): - c = language.c - assert parse_line(c, '" a \\\n', code_only=True) == ('" a \\\n', make_state('', None, '"')) - - -def test_previous_state_maintained(): - c = language.c - assert parse_line(c, '', ['foo']) == ('', make_state('', ['foo'])) - assert parse_line(c, '/**/', ['foo']) == ('/**/', make_state('', ['foo'])) - assert parse_line(c, '/*', ['foo']) == ('/*', make_state('', ['foo'])) - j = language.java - assert parse_line(j, '/*', ['foo']) == ('/*', make_state('', ['foo', '*/'])) - - -def test_code_of_resumed_multiline_comment(): - c = language.c - assert parse_line(c, 'a', ['*/'], True) == (' ', make_state('', ['*/'])) - assert parse_line(c, 'a */', ['*/'], True) == (' ', make_state('', [])) - - -def parse_lit_line(lang, line, code_only=False): - return safe_parse_line(lang, make_state(line, None, lang.string_literal_start), code_only=code_only) - - -def test_previous_state_maintained_literal(): - c = language.c - assert parse_lit_line(c, '') == ('', make_state('', in_literal='"')) - assert parse_lit_line(c, '"') == (' ', make_state('')) - - -def test_code_of_resumed_multiline_literal(): - c = language.c - assert parse_lit_line(c, '', True) == ('', make_state('', in_literal='"')) - assert parse_lit_line(c, '"', True) == ('"', make_state('')) - - -def c_comments(s, keep_tokens=True): - return list(rfc.parse_file(language.c, StringIO(s), keep_tokens=keep_tokens)) - - -def test_parse_file(): - assert c_comments('/* hello */ world\n') == ['/* hello */ \n'] - assert c_comments('/* hello */ world\n', False) == [' hello \n'] - - -def test_parse_comments_via_reduce(): - def f(st, x): - st.line = x - _, st = safe_parse_line(language.c, st) - return st - - assert reduce(f, ['/*a', 'b*/'], make_state()) == make_state() - assert reduce(f, ['/*a', 'b'], make_state()) == make_state('', ['*/']) -- 2.37.1 (Apple Git-137.1) _______________________________________________ devel mailing list devel@rtems.org http://lists.rtems.org/mailman/listinfo/devel
