将文本文件中的英文单词找出来（进行分词），并统计每个单词出现的次数。

import re
def get_char(txt):  # 定义分词函数
    vlist = re.split('[,;:."\s]\s*',txt)  # 依据分词符用split函数分词
    print(vlist)
    vdic_frequency = dict()  # 创建一个字典
    print(vdic_frequency)
    for vchar in vlist:  # 遍历所有字符，并统计字符出现的个数
        if vchar in vdic_frequency:  # 判断是否在字典中
            vdic_frequency[vchar]+=1 如果在字典中，则次数加1
        else:
            vdic_frequency[vchar] = 1不在字典中，说明第一次出现，次数为1
    print(vdic_frequency)
    vdic_sort = sorted(vdic_frequency.items(),key=lambda item: item[1],reverse=True)  # 排序.items()将字典转化为列表中元组的形式，lambda对item进行操作，item[0]表示键名，item【1】表示键值，这里排序依据是键值。逆序排序
    return vdic_sort

if __name__ == '__main__':
    with open('F:\研究生\python-100例\\test.txt','r') as f:# \t表示转义字符，\\t才能读取地址
        vtext = f.read()
    vstr = get_char(vtext)
    print(vstr)

文本分词的介绍网上已经很全面了，这里主要介绍一种文本分词、去停用词的具体实现，停用词表是对百度、哈工大等常见停用词表融合后去重

import csv
import datetime
import re
import pandas as pd
import numpy as np
import jieba

# 停用词路径
def stopwordslist(filepath):
    stopwords = [line.strip() for line in open(filepath, 'r', encoding='utf-8').readlines()]
    return list(set(stopwords))


# 分词去停用词
def seg_sentence(sentence):
    sentence_seged = jieba.cut(sentence.strip())
    stopwords = stopwordslist(r"C:\Users\lenovo\Desktop\fin_data\NLP_code\wordtoremove.txt")  # 这里加载停用词的路径
    
    outstr = ''
    for word in sentence_seged:
        if word not in stopwords:
            if word != '\t':
                outstr += word
                outstr += " "
    return outstr


# 分词、去停用词、只保留动 名词
def seg_sentence_return_vn(sentence):
    seg = psg.cut(sentence.strip())
    
    outstr = ''
    for x in seg:
        if x.flag in ('n','v'):
            outstr += x.word
            outstr += " "
    return outstr 


if __name__ == '__main__':  
    
    filename = 'zhihu_data_setiment.csv'
    csv_data = pd.read_csv(filename, header = 0, index_col = False, engine='python',encoding = 'utf-8-sig')
    
    file_userdict = 'personal_dic.txt'
    jieba.load_userdict(file_userdict)
    
    for i in range(csv_data.shape[0]):
        answer = str(csv_data.iloc[i,14])
        csv_data.iloc[i,20] = seg_sentence(answer)
        csv_data.iloc[i,21] = seg_sentence_return_vn(answer)
        
        # 进度查看器
        if(i % 500 ==0):
            print(i,end = '  ')
        
    # 写文件
    csv_data.to_csv("zhihu_data_jieba_seg.csv",header=True,index=False,encoding='utf-8-sig')