python常用文本串处理库学习笔记

def pre_process(text):
    """
    文本预处理:
    1. 删除掉符号
    2. 大写转小写s
    :param text:
    :return:
    """
    text = del_pun(text)  # 删除符号
    text = text.lower()  # 英文转小写
    text = text2list(text)  # 字符串转序列
    # todo 将itn转换过来
    return text


def is_equal(ref,hyp):
    """
    比较两个短文本是否等价
    :param ref:
    :param hyp:
    :return:
    """
    ref = del_pun(ref)  # 删除符号
    ref = ref.lower()  # 英文转小写

    hyp = del_pun(hyp)  # 删除符号
    hyp = hyp.lower()  # 英文转小写

    if ref == hyp:
        return True
    else:
        return False

def del_pun(text):
    """
    删除文本中的特殊符号
    :param text:
    :return:
    """
    list_pun = (",",
                "，",
                "。",
                "《",
                "》",
                "“",
                "”",
                "?",
                "？",
                "(",
                ")",
                "（",
                "）",
                "!",
                "！",
                "：",
                ";",
                "；",
                "……",
                "…",
                "、",
                "@",
                "[",
                "]",
                "_",
                "*",
                "-",
                "&",
                "×",
                "·",
                "t",
                "n")  # 需要过滤的标点:×
    stra = ''
    for word in text:
        if word in list_pun:
            continue
        else:
            stra += word
    stra = stra.strip()  # 去掉首尾空格
    return stra


def is_upper_eng_char(uchar):
    """
    unicode大写字母
    :param uchar:
    :return:
    """
    if 'u0041' <= uchar <= 'u005a':
        return True
    return False


def is_lower_eng_char(uchar):
    if 'u0061' <= uchar <= 'u007a':
        return True
    return False


def is_eng_char(uchar):
    """
    字符是否是英文,英文的Unicode编码
    :param uchar:
    :return:
    """
    if is_upper_eng_char(uchar) or is_lower_eng_char(uchar):
        return True
    return False


def is_num_char(uchar):
    return True if 'u0030' <= uchar <= 'u0039' else False


def is_cjk_char(uchar):
    """
    文本中是否包含非中文,CJK Unified Ideographs (4E00–9FFF) wiki
    汉字的unicode范围是：0x4E00~0x9FA5
    :param uchar:
    :param text:
    :return:
    """
    return True if 'u4E00' <= uchar <= 'u9FA5' else False


def hasNum(text):
    """
    文本中是否包含数字
    :param text:
    :return:
    """
    for uchar in text:
        if is_num_char(uchar):
            return True
    return False


def text2list(text):
    """
    将文本的每个字符转换成list，英文单词算一个字符，以空格作为区分
    :param text:
    :return:
    """
    tlist = []
    lastEng = False
    begin = 0
    for i in range(len(text)):
        # print(text[i], isEng(text[i]))

        if i == (len(text) - 1):
            if lastEng and is_eng_char(text[i]):
                tlist.append(text[begin:])
            elif lastEng and not is_eng_char(text[i]):
                tlist.append(text[begin:i])
                tlist.append(text[i])
            else:
                tlist.append(text[i])
            break

        if lastEng:
            if not is_eng_char(text[i]):
                tlist.append(text[begin:i].strip())
                if text[i] != ' ':
                    tlist.append(text[i])
                lastEng = False
        else:
            if is_eng_char(text[i]):
                begin = i
                lastEng = True
            else:
                if text[i] != ' ':
                    tlist.append(text[i])

    return tlist


class MyTest(TestCase):

    def test_hasNum(self):
        self.assertTrue(hasNum('ab3de好不好'), msg='应该存在数字')
        self.assertFalse(hasNum('ab不好'), msg='应该不存在数字')


if __name__ == '__main__':
    print(del_pun('AB-C'))