情景

你需要在Mac下复制文件到文件夹“B”，但是有部分文件重复了，这时系统没有“保留二者”的选项，只能"跳过"，"停止"，"替换"，这个简单的要求"保留"为什么不做出来？APP store里面有收费的软件针对这个小问题，¥30。。。有必要么？

以下脚本是我开发出来，Mac端合并重复文件夹、文件用的，欢迎一起探讨。

 

效果图「2」





 

准备材料

本工具（mergeFolders.php） https://download.csdn.net/download/qq285744011/12235772
	
桌面新建两个文件夹，from和to，你需要把你要合并的所有文件夹、文件都扔进去from
 

使用方法

 * 【使用方法】

     * 1. 在桌面上新建两个文件夹，取名"from"和"to"，把你需要合并的文件夹、文件统统复制到文件夹"from"

     * 2. 下载本文件"mergeFolders.php"到Macbook上的桌面

     * 3. 打开命令行工具 （按F4 > 文件夹"其它" > 终端）

     * 4. 在Mac中开启PHP语言支持： https://rudon.blog.csdn.net/article/details/104745975

     * 5. 在命令行中，输入命令：php ~/Desktop/mergeFolders.php

     * 6. OK, 稍等片刻，检查文件夹"to"是否包含了所有的文件

 

 

脚本分享

<?php


    /**
     * 主题：文件夹合并工具 for Mac， 实现复制文件时"保留两个文件"的效果
     * 
     * 作者：Rudon <285744011@qq.com>
     * 创建：2020-03-09
     * 描述：
     *      在Windows系统下从-文件夹A-复制文件到-文件夹B-时，
     *      如果文件logo.jpg重复了，系统会在文件夹B里创建文件'logo(1).jpg'，
     *      但是，Mac系统不会，只能"跳过"，"停止"，"替换"，这个简单的要求"保留"为什么不做出来？
     *      本工具支持多层文件夹合并。
     *      
     * 
     * 【使用方法】
     * 1. 在桌面上新建两个文件夹，取名"from"和"to"，把你需要合并的文件夹、文件统统复制到文件夹"from"
     * 2. 下载本文件"mergeFolders.php"到Macbook上的桌面
     * 3. 打开命令行工具 （按F4 > 文件夹"其它" > 终端）
     * 4. 在Mac中开启PHP语言支持： https://rudon.blog.csdn.net/article/details/104745975
     * 5. 在命令行中，输入命令：php ~/Desktop/mergeFolders.php
     * 6. OK, 稍等片刻，检查文件夹"to"是否包含了所有的文件
     * 
     */

    /* summary.xls => summary(1).xls */
    define('SYMBOL_LEFT',   '(');
    define('SYMBOL_RIGHT',   ')');
    define('FOLDER_FROM',   'from');
    define('FOLDER_TO',     'to');
    
    
    
    
    function a ($v){
        header('Content-Type: text/css; charset=utf-8');
        print_r($v);
        die();
    }
    
    
    $action = new mergeFolder();
    $action->merge();
    $action->echoResult();
    
    

    class mergeFolder {
        public $working_path;
        public $from;
        public $to;
        public $merged_list;
        public $special_cases;
        public $need_log;


        public function __construct() {
            /* 当前文件夹 */
            $this->working_path = dirname(__FILE__).'/';
            $this->from = $this->working_path . FOLDER_FROM . '/';
            $this->to = $this->working_path . FOLDER_TO . '/';
            $this->merged_list = array();
            $this->special_cases = array();
            $this->need_log = true;
            
            $this->check_folders();
        }
        
        public function check_folders () {
            if (!is_dir($this->from)) {
                $this->echoError('找不到文件夹`'.FOLDER_FROM.'`');
            }
            if (!is_dir($this->to)) {
                $this->echoError('找不到文件夹`'.FOLDER_TO.'`');
            }
        }
        
        public function echoError ($message) {
            die( PHP_EOL .'ERROR: '. $message . PHP_EOL );
        }
        
        
        public function get_list_of_files_recursive ($folder) {
            $return = array();
            
            $folder = rtrim($folder, '/').'/';
            if(is_dir($folder)){
                $list = scandir($folder);
                foreach ($list as $oneFile) {
                    if (!preg_match('/^\./', $oneFile)) {
                        /* 除掉那些点号开头的文件 */
                        if (is_dir($folder. $oneFile)) {
                            $sub_files = $this->get_list_of_files_recursive($folder.$oneFile);
                            $return = array_merge($return, $sub_files);
                        } else {
                            $return[] = $folder . $oneFile;
                        }
                    }
                }
            }
            
            return $return;
        }
        
        
        public function merge () {
            $files = $this->get_list_of_files_recursive($this->from);
            $counts = count($files);
            if(!$counts){
                $this->echoError('没有任何文件可合并');
            }
            
            foreach ($files as $k => $one) {
                /* 对比文件名 */
                $file_name = pathinfo($one, PATHINFO_BASENAME);
                $tobe = $this->to . $file_name;
                $file_ext = pathinfo($one, PATHINFO_EXTENSION);
                $file_name_pre = pathinfo($one, PATHINFO_FILENAME);
                
                if (!in_array($file_name, $this->merged_list)) {
                    /* 不重复 */
                    $this->merged_list[] = $file_name;
                    copy($one, $tobe); // 复制
                    unlink($one);  // 删除
                    
                    
                } else {
                    $this->log('Repeat: '.$file_name.' in '.$one);
                    
                    /* 重复文件 */
                    $zuo = SYMBOL_LEFT;
                    $you = SYMBOL_RIGHT;
                    
                    if (!key_exists($file_name, $this->special_cases)) {
                        /* 新猪肉 */
                        $this->special_cases[$file_name] = 1;
                        
                    } else {
                        /* 旧猪肉 */
                        $lastNum = $this->special_cases[$file_name];
                        $newNum = $lastNum + 1;
                        $this->special_cases[$file_name] = $newNum;
                    }
                    
                    $repeat_num = $this->special_cases[$file_name];
                    $file_name_new = $file_name_pre.$zuo.$repeat_num.$you.'.'.$file_ext;
                    
                    $this->log('Repeat: '.$file_name.' [Repeat] '.$repeat_num);
                    $this->log('Repeat: '.$file_name.' [To Be] '.$file_name_new);
                    
                    while (in_array($file_name_new, $this->merged_list)) {
                        $this->log('Repeat: '.$file_name.' [Exists in Target folder] yes');
                        
                        $repeat_num++;
                        $this->special_cases[$file_name] = $repeat_num;
                        $file_name_new = $file_name_pre.$zuo.$repeat_num.$you.'.'.$file_ext;
                        
                        $this->log('Repeat: '.$file_name.' [New Repeat] '.$repeat_num);
                        $this->log('Repeat: '.$file_name.' [New To Be] '.$file_name_new);
                    }
                    
                    $this->merged_list[] = $file_name_new;
                    copy($one, $this->to . $file_name_new); // 复制
                    unlink($one);  // 删除
                    
                    $this->log('Repeat: '.$file_name.' [Done] ');
                }
            }
            
        }
        
        public function log ($mess) {
            if ($this->need_log) {
                echo PHP_EOL.'[DEV] '.$mess.PHP_EOL;
            }
        }
        
        public function echoResult () {
            die( PHP_EOL .'成功！其中重复文件共有'. count($this->special_cases) .'组' . PHP_EOL );
        }
        
    } /* End of CLASS */
    
    
    
    

 

本次NLP作业需要每个人在小组选定领域下进行子领域词典制作，我们小组选定的领域为动物。我个人选定的子领域为昆虫，原始语料库来自《昆虫记》这本书。通过爬虫或者复制粘贴可以在本地得到关于《昆虫记》的文本文件。
数据的处理

读取文本，将句号替换成换行，跳过空行
通过自建筛选字典和清华动物字典，对文本进行处理，保留每行含有动物词汇的行
按照7:3的比例，划分训练集和测试集
读取训练集，生成昆虫领域词典。（most_common可以指定返回数目，因为有些动物名只出现一次，并且生成词典较小。所以这里并未指定，而是全部返回，然后人工筛选）
用jeiba对训练集进行分词处理

评估
中文分词系统的评价指标一般有切分准确率(Precision)、召回率(Recall)、分词精度(F1-Measure)等。本实验将F1作为评测指标来评价HMM模型对昆虫领域的分词效果，F1的计算公式如下所示：

$\begin{aligned}
P&=\frac{系统正确识别的词语个数}{系统识别的词语个数}\times100\%
\\
\\
R&=\frac{系统正确识别的词语个数}{测试语料中所有的词语个数}\times100\% 
\\
\\
F1&=\frac{2×P×R}{P+R}(P+R)\times100\%
\end{aligned}$
python程序
import os
import sys
import random
import json
import jieba
import jieba.posseg
from collections import Counter


class ProcessData(object):
    def __init__(self, *dic_path):
        self.dictionary = set()
        self.maximum = 0
        # 加载词典
        for path in dic_path:
            self.load_dic(path)

    # 加载字典
    def load_dic(self, dic_path):
        with open(dic_path, 'r', encoding='utf-8') as fp:
            for line in fp:
                line = line.strip().split()[0]
                if not line:
                    continue
                self.dictionary.add(line)
                self.maximum = max(self.maximum, len(line))

    # 判断text中是否包含词典中的词
    def return_data(self, text):
        index = 0
        while index < len(text):
            match = False
            for size in range(self.maximum, 0, -1):
                if index + size > len(text):
                    continue
                piece = text[index:(index + size)]
                if piece in self.dictionary:
                    return True
            if not match:
                index += 1
        return False

    # 划分数据集和测试集
    def train_test_split(self, data_path, train_path, test_path):
        try:
            with open(data_path, 'r', encoding='utf-8') as fp, open(
                    train_path, 'w',
                    encoding='utf-8') as train, open(test_path,
                                                     'w',
                                                     encoding='utf-8') as test:
                lines = fp.readlines()
                threshold = int(len(lines) * 0.7)
                random.shuffle(lines)

                for i, line in enumerate(lines):
                    if i < threshold:
                        train.write(line)
                    else:
                        test.write(line)
        except Exception:
            print(sys.stderr, "文件读写出现错误")
            raise Exception
            sys.exit(1)

    # 对text文件进行词性标注
    def postag_txt(self, inputFile, outputFile):
        with open(inputFile, 'r',
                  encoding="utf-8") as fin, open(outputFile,
                                                 'w+',
                                                 encoding="utf-8",
                                                 newline='') as fout:
            # 词性统计
            d = {}
            for eachLine in fin:
                # 跳过空行
                if not len(eachLine.strip()):
                    continue
                else:
                    line = eachLine.strip().replace('。', os.linesep)
                    # 使用jieba词性标注
                    posLine = jieba.posseg.cut(line)
                    newLine = ''
                    for key in posLine:
                        # newLine += "{}/{}  ".format(key.word, key.flag)
                        d[key.flag] = d.get(key.flag, 0) + 1
                    # fout.write(newLine + os.linesep)
            fout.write(json.dumps(d))
        return True

    # 词频统计
    def count_words(self, text):
        # 加载停用词表
        stop_path = os.path.join(sys.path[0], r'.\data\stop_words.utf8')
        stopwords = [
            line.strip()
            for line in open(stop_path, 'r', encoding='utf-8').readlines()
        ]
        seg_list = jieba.cut(text)
        c = Counter()
        for word in seg_list:
            if word not in stopwords and len(word) > 1 and word != os.linesep:
                c[word] += 1

        # 前count
        lines = ""
        for (word, num) in c.most_common():
            line = word + "  " + str(num) + os.linesep
            lines += line
        return lines

    # jeiba分词
    def seq_word(self, input_path, output_path):
        with open(input_path, 'r',
                  encoding='utf-8') as fin, open(output_path,
                                                 'w',
                                                 encoding='utf-8') as fout:
            for line in fin.readlines():
                seq_list = jieba.cut(line, cut_all=False)
                fout.write(' '.join(seq_list))
        print("jeiba分词完成")


def Process():

    dict1_path = os.path.join(sys.path[0],
                              r'.\data\THUOCL_animal.txt')  # 清华动物词典
    dict2_path = os.path.join(sys.path[0], r'.\data\my_animal.txt')  # 自建词典

    input_path = os.path.join(sys.path[0], r'.\data\insect_origin.txt')
    output_path = os.path.join(sys.path[0], r'.\data\insect.txt')

    train_path = os.path.join(sys.path[0],
                              r'.\data\train_insect.txt')  # 生成训练集路径

    test_path = os.path.join(sys.path[0], r'.\data\test_insect.txt')  # 生成测试集路径

    pro = ProcessData(dict1_path, dict2_path)  # 加载词典

    try:
        with open(input_path, 'r',
                  encoding='utf-8') as input_text, open(output_path,
                                                        'w',
                                                        encoding='utf-8',
                                                        newline='') as output:
            for line in input_text:
                flag = pro.return_data(line.strip())
                if flag:
                    print("line:", line)
                    output.write(line)
    except Exception:
        print(sys.stderr, "文件打开错误")
        raise Exception
        sys.exit(1)

    print("数据处理完成")

    pro.train_test_split(output_path, train_path, test_path)
    print("训练集和测试集生成")

    my_dict = os.path.join(sys.path[0], r'.\data\my_dict.txt')
    # 词典生成
    with open(train_path, 'r',
              encoding='utf-8') as fin, open(my_dict,
                                             'w',
                                             encoding='utf-8',
                                             newline="") as fout:
        text = fin.read()
        text = pro.count_words(text)
        fout.write(text)

    # 用jeiba对训练集进行分词处理
    jieba_train = os.path.join(sys.path[0], r'.\data\jieba_train.txt')
    pro.seq_word(train_path, jieba_train)

    # 用jeiba对测试集进行分词处理
    jieba_train = os.path.join(sys.path[0], r'.\data\jieba_test.txt')
    pro.seq_word(test_path, jieba_train)

    output_tag = os.path.join(sys.path[0], r'.\data\train_tag.txt')
    pro.postag_txt(train_path, output_tag)

# F1值计算
def estimate_F1():
    # 评估分词模型的准确率
    hmm_path = r'.\data\hmm_output.txt'
    jieba_path = r'.\data\jieba_test.txt'

    with open(hmm_path, 'r',
              encoding='utf-8') as f_hmm, open(jieba_path,
                                               'r',
                                               encoding='utf-8') as f_jieba:
        all_words_answer = 0
        all_words_sample = 0
        correct = 0
        sentence1 = f_hmm.readline()
        sentence2 = f_jieba.readline()

        while sentence1:
            hmm_line = sentence1.split()
            jieab_line = set(sentence2.split())
            same_list = [x for x in hmm_line if x in jieab_line]

            all_words_answer += len(hmm_line)
            all_words_sample += len(jieab_line)
            correct += len(same_list)

            sentence1 = f_hmm.readline()
            sentence2 = f_jieba.readline()

            recall = correct / all_words_answer
            precision = correct / all_words_sample
            f_mesure = (2 * precision * recall) / (precision + recall)
        print("词数：", all_words_answer)
        print("Precision:", round(precision, 4), "Recall", round(recall, 4),
              "F-mesure", round(f_mesure, 4))


if __name__ == "__main__":
    Process()

清华动物词典



自建筛选词典



stop_words文件

insect.txt文件



生成词典my_dict