Python：tokenize 库高级用法举例和应用详解

import tokenize  # 导入tokenize库
from io import BytesIO  # 导入BytesIO模块，用于模拟文件对象

# 示例Python代码
code = "print('Hello, World!')"

# 将字符串转为字节对象，tokenize模块需要字节对象输入
code_bytes = BytesIO(code.encode('utf-8'))

# 使用tokenize.tokenize方法解析代码
tokens = tokenize.tokenize(code_bytes.readline)

# 逐个打印标记
for token in tokens:
    print(token)  # 输出标记包含类型、字符串、位置等信息

import tokenize
from io import BytesIO

# 示例Python代码
code = """
# This is a comment
print('Hello, World!')  # This is another comment
"""

# 将字符串转为字节对象
code_bytes = BytesIO(code.encode('utf-8'))

# 使用tokenize生成标记
tokens = tokenize.tokenize(code_bytes.readline)

# 统计注释行的数量
comment_count = 0

for token in tokens:
    if token.type == tokenize.COMMENT:
        comment_count += 1  # 增加注释行计数器

print(f'Total number of comment lines: {comment_count}')  # 输出注释行数量

import tokenize
from io import BytesIO

# 示例Python代码
code = """
def add(a, b):
    return a + b
"""
# 将字符串转为字节对象
code_bytes = BytesIO(code.encode('utf-8'))

# 使用tokenize生成标记
tokens = tokenize.tokenize(code_bytes.readline)

formatted_code = []

for token in tokens:
    if token.type == tokenize.INDENT:
        formatted_code.append(' ' * 4)  # 替换缩进为4个空格
    elif token.type == tokenize.NAME:
        formatted_code.append(token.string.upper())  # 将变量名转为大写
    else:
        formatted_code.append(token.string)

# 重新组合并输出格式化后的代码
formatted_code_str = ''.join(formatted_code)
print(formatted_code_str)