介绍：

笔趣阁小说TXT采集工具是一款非常好用的笔趣阁小说下载工具，能够帮助用户轻松搜索采集你喜欢的小说，支持完本下载，方便你在本地离线阅读，且无任何广告，感兴趣的用户欢迎下载使用！

网盘下载地址：

http://kekewl.cc/bwwheykkw1N0

图片：

今天来实现以下笔趣阁小说爬虫
笔趣阁的小说爬取难度还是比较低的（不涉及搜索功能）
咱们用requests和xpath来完成这个小爬虫

首先肯定是导包

import requests
import time
from lxml import etree

然后来写两个辅助函数
分别用于请求网页和xpath解析
函数会让我们后面的程序更加简洁方便

def get_tag(response, tag):
    html = etree.HTML(response)
    ret = html.xpath(tag)
    return ret


def parse_url(url):
    response = requests.get(url)
    response.encoding = 'gbk'
    return response.text

我们是从目录页开始爬取的
所以首先我们要获取目录页面里面所有的章节网址

def find_url(response):
    chapter = get_tag(response, '//*[@id="list"]/dl/dd/a/@href')
    # print(chapter)
    for i in chapter:
        url_list.append(start_url + i)
        # url_list.append('https://www.52bqg.com/book_187/' + i)
    # print(url_list)

然后根据章节的网址
去获取章节内的信息

def find_content(url):
    response = parse_url(url)
    chapter = get_tag(response, '//*[@id="box_con"]/div[2]/h1/text()')[0]
    content = get_tag(response, '//*[@id="content"]/text()')
    print('正在爬取', chapter)
    with open('{}.txt'.format(title), 'at', encoding='utf-8') as j:
        j.write(chapter)
        for i in content:
            if i == '\r\n':
                continue
            j.write(i)
    j.close()
    print(chapter, '保存完毕')
    time.sleep(2)

最后就是一个主体函数

def main():
    global title, start_url
    # start_url = 'https://www.52bqg.com/book_187/'
    start_url = input('输入爬取小说的目录页:例如（https://www.52bqg.com/book_187/）')
    response = parse_url(start_url)
    # print(response)
    title = get_tag(response, '//*[@id="info"]/h1/text()')[0]
    # print(title)
    find_url(response)
    # print(1)
    for url in url_list:
        find_content(url)

完整代码如下：

import requests
import time
from lxml import etree


url_list = []


def get_tag(response, tag):
    html = etree.HTML(response)
    ret = html.xpath(tag)
    return ret


def parse_url(url):
    response = requests.get(url)
    response.encoding = 'gbk'
    return response.text


def find_url(response):
    chapter = get_tag(response, '//*[@id="list"]/dl/dd/a/@href')
    # print(chapter)
    for i in chapter:
        url_list.append(start_url + i)
        # url_list.append('https://www.52bqg.com/book_187/' + i)
    # print(url_list)


def find_content(url):
    response = parse_url(url)
    chapter = get_tag(response, '//*[@id="box_con"]/div[2]/h1/text()')[0]
    content = get_tag(response, '//*[@id="content"]/text()')
    print('正在爬取', chapter)
    with open('{}.txt'.format(title), 'at', encoding='utf-8') as j:
        j.write(chapter)
        for i in content:
            if i == '\r\n':
                continue
            j.write(i)
    j.close()
    print(chapter, '保存完毕')
    time.sleep(2)


def main():
    global title, start_url
    # start_url = 'https://www.52bqg.com/book_187/'
    start_url = input('输入爬取小说的目录页:例如（https://www.52bqg.com/book_187/）')
    response = parse_url(start_url)
    # print(response)
    title = get_tag(response, '//*[@id="info"]/h1/text()')[0]
    # print(title)
    find_url(response)
    # print(1)
    for url in url_list:
        find_content(url)


if __name__ == '__main__':
    main()

效果图：

笔趣阁小数的爬虫还是比较好写的
下篇文章我们为这个爬虫加上GUI界面
用Python实现笔趣阁小说爬取 GUI版

好了，本文到此结束。如果对编程、计算机、程序员方面感兴趣的话，欢迎私信联系我，随时交流！点个关注，是对我莫大的鼓励！
我们这边有一个Python交流群，里面程序源码、教程视频，还有大佬解答问题，群号：821189983。

java爬虫爬取笔趣阁小说

package novelCrawler;

import org.jsoup.Connection;
import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import ui.DownMsgUI;
import ui.crawlerUI;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.Timer;
import java.util.TimerTask;

import javax.sound.sampled.AudioFormat.Encoding;
import javax.swing.JFrame;
import javax.swing.JOptionPane;
import javax.swing.JProgressBar;
public class biquge {

	   public biquge(String nn,String start,JProgressBar jpbVal,JProgressBar jpb2) {
	        long t1 = System.currentTimeMillis();
	        
	        //查找小说
	        Connection connection1 = Jsoup.connect("http://www.xbiquge.la/xiaoshuodaquan/");
	        
	        Document document1 = null;
			try {
				document1 = connection1.get();
			} catch (IOException e2) {
				// TODO 自动生成的 catch 块
				e2.printStackTrace();
			}
	        Elements elementsLis1=null;
			try {
	        Element elementUL1 = document1.select("[class=novellist]").first();
	        elementsLis1 = elementUL1.select("li");
			} catch (Exception e1) {
				// TODO 自动生成的 catch 块
				e1.printStackTrace();
			}
			jpbVal.setVisible(true);
			int count = 0;
			 DownMsgUI dmu = new DownMsgUI();//消息框
	        //巡官遍历获取到的整个elementsLis集合
	        for(Element elementLi1 : elementsLis1){
	        	
	        	try {
					Thread.sleep(200);
				} catch (InterruptedException e1) {
					// TODO 自动生成的 catch 块
					e1.printStackTrace();
				}
	        	
	            Element elementA1 = elementLi1.select("a").first();      
	            String href = elementA1.attr("href");//获取标签中的属性值（它这里采用的是相对路径的写法
	            String novelNa = elementA1.text();//小说名
	            
	            
	            jpbVal.setMaximum(elementsLis1.size());
	            
	          
	            
	            int result1 = novelNa.indexOf(nn);//匹配小说名
	            if(result1 != -1){
	            	//跟进小说
	            	jpbVal.setValue(elementsLis1.size());
	            	//JOptionPane msg = new JOptionPane();
	            	//msg.setBounds(100, 100, 100, 100);
	            	//JOptionPane.showMessageDialog(jpb2,"已找到小说："+novelNa);
	            	dmu.ta.append("找到小说： "+novelNa+"\n");
	            	connectNovel(novelNa,href,start,jpb2,dmu);
	            	break;
	            	
	            }else{
	            	//继续往下寻找
	            	//System.out.println("未找到小说!");
	            	jpbVal.setValue(count);
	            }
	            count++;
	            if(count>=elementsLis1.size()) {
	            	System.out.println("未找到小说!");
	            }
	           
	    }
	   }    
	        
	        
	    /    
	  public void connectNovel(String novelNa,String h,String start,JProgressBar jpb2,DownMsgUI dmu) {   
	        //1.与我们要爬取数据的页面建立连接
	       Connection connection = Jsoup.connect(h);
	        
	        jpb2.setVisible(true);

	        Document document = null;
			try {
				document = connection.get();
			} catch (IOException e2) {
				// TODO 自动生成的 catch 块
				e2.printStackTrace();
			}
	        
	 
	  
	        Element elementUL = document.select("[id=list]").first();
	        Elements elementsLis = elementUL.select("dd");

	     //小说信息
	        Element el = document.select("[id=info]").first();
	        Elements ele = el.select("p");
	        dmu.ta.append(ele.text().substring(0, ele.text().indexOf("动"))+"\n");
	        String str1=ele.text().substring(0, ele.text().indexOf("部"));
	        String str2=ele.text().substring(str1.length()+1, ele.text().length());
	        dmu.ta.append(str2+"\n");
	        System.out.println(str2);
	        
	        int midTime = elementsLis.size();//单位s
	        //所需时间
	        jpb2.setMaximum(midTime);
	        //residueTime(midTime);//倒计时
	        
	        
	       int  count2=0;
	      
	          boolean flag = false;
	          
	          int  count=0;
	          int count3=0;
	          jpb2.setMaximum(elementsLis.size());
	        //巡官遍历获取到的整个elementsLis集合
	        for(Element elementLi : elementsLis){
	        	try {
					Thread.sleep(10);
				} catch (InterruptedException e1) {
					// TODO 自动生成的 catch 块
					e1.printStackTrace();
				}
	        	
	            Element elementA = elementLi.select("a").first();      
	            String href = elementA.attr("href");//获取标签中的属性值（它这里采用的是相对路径的写法）

	            String imgName = elementA.text();
	            	
	            	count3++;
	          
	   	     

	            int result1 = imgName.indexOf(start);//匹配章节
	            if(result1 != -1){
	            	//开始下载
	            	
	            	try {
						download(novelNa,href,imgName, dmu);
						count++;
						
						jpb2.setMinimum(count3);
						jpb2.setValue(jpb2.getMinimum()+(count2++));
					} catch (IOException e) {
						// TODO 自动生成的 catch 块
						e.printStackTrace();
					}
	            	flag = true;
	            }else{
	            	//继续往下寻找
	            	if(flag == true) {
	            		try {
							download(novelNa,href,imgName, dmu);
							count++;
							jpb2.setValue(jpb2.getMinimum()+(count2++));
						} catch (IOException e) {
							// TODO 自动生成的 catch 块
							e.printStackTrace();
						}
	            	}
	            }

	           if(count3==elementsLis.size())
	           {
	        	   dmu.ta.append("下载完成     共  "+count+" 章");
	        	   jpb2.setValue(jpb2.getMaximum());
	           }
	    }
	        
	        
	   }    

	    public static void download(String novelNa,String href,String imgName,DownMsgUI dmu) throws IOException {
	    	 String netPath = "http://www.xbiquge.la"+href;

	            Connection newConnection = null; 
	            Document newDocument  = null; 
	            
	            try {
					Thread.sleep(100);
					 
	                 newConnection = Jsoup.connect(netPath);
	                 Thread.sleep(500);
	                  newDocument = newConnection.get();
				} catch(HttpStatusException e) {
					System.out.println("下载错误,尝试重新连接");
					 try {
							Thread.sleep(500);
							 
			                 newConnection = Jsoup.connect(netPath);
			                 Thread.sleep(1000);
			                  newDocument = newConnection.get();
						} catch(HttpStatusException e1) {
							System.out.println("下载错误2,尝试重新连接");
							
						} catch
			            (InterruptedException e3) {
							
							e.printStackTrace();
						}
					
				}
	            catch
	            (InterruptedException e) {
				
					e.printStackTrace();
				}
	          
	            
	            try {
					Thread.sleep(500);
				} catch (InterruptedException e1) {
					
					e1.printStackTrace();
				}
	            
	            Element div = newDocument.select("[id=content]").first();
	         
	            String divStyle = div.attr("style");
	            String text = div.text().trim();
	            text = new String(text.getBytes("UTF-8"),"UTF-8");
	            String l = System.getProperty("line.separator");
	            
	        text = text.replace("。", "。"+l);
	       // System.out.print(text);
	            System.out.print("正在下载："+imgName+"    ");
	            FileOutputStream fileOutputStream = new FileOutputStream("./"+novelNa+".txt",true);
	            fileOutputStream.write(l.getBytes("UTF-8"));
	            fileOutputStream.write(imgName.getBytes("UTF-8"));
	            fileOutputStream.write(l.getBytes("UTF-8"));
	            fileOutputStream.write(text.getBytes("UTF-8"));//不指定则空格乱码，iso编码
	            fileOutputStream.flush();
	            fileOutputStream.close();
	            System.out.println("下载完成");
	           
	        	dmu.ta.append("正在下载："+imgName+"    \n");
	        	
	        long t4 = System.currentTimeMillis();
	        //double time = (double) (t4-t1)/1000;
	       // System.out.println("恭喜您已完成全部下载，共耗时："+time+"秒，下载"+"章");
	    }

	    
	    
}

	

```java
package novelCrawler;


import java.io.BufferedReader;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileOutputStream;

import java.io.InputStreamReader;

import java.io.OutputStreamWriter;

import java.net.URL;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

 

public class GetText {

/**

* 1、根据小说存放位置创建file对象

2、根据网页结构编写正则，创建pattern对象

3、编写循环，创建向所有小说章节页面发起网络请求的url对象

4、网络流BufferReader

5、创建输入流

6、循环读取请求得到的内容，使用正则匹配其中的内容

7、将读取到的内容写入本地文件，知道循环结束

8、注意代码中的异常处理

 

* @param args

*/

public static void main(String[] args) {

// 1、根据小说存放位置创建file对象

File file = new File("D:\\File\\three_guo.txt");

// 2、根据网页结构编写正则，创建pattern对象

String regex_content = "<p.*?>(.*?)</p>";

String regex_title = "<title>(.*?)</title>";

 

Pattern p_content = Pattern.compile(regex_content);

Pattern p_title = Pattern.compile(regex_title);

 

Matcher m_content;

Matcher m_title;

 

// 3、编写循环，创建向所有小说章节页面发起网络请求的url对象

for (int i = 1; i <= 120; i++) {

System.out.println("第" + i + "章开始下载。。。");

try {

// 创建每一个页面的url对象

URL url = new URL("http://www.xbiquge.la/13/13959/");

// 创建网络读取流

BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(),"utf8"));

 

// 4、读取网络内容网络流BufferReader

String str = null;

 

// 5、创建输入流

BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file,true)));

 

while ((str = reader.readLine()) != null) {

m_title = p_title.matcher(str.toString());

m_content = p_content.matcher(str.toString());

 

// 获取小说标题并写入本地文件

Boolean isEx = m_title.find();

if (isEx) {

String title = m_title.group();

// 清洗得到的数据

title = title.replace("<title>", "").replace("</title>", "");

System.out.println(title);

writer.write("第" + i + "章：" + title + "\n");

}

 

while (m_content.find()) {

String content = m_content.group();

// 清洗得到的数据

content = content.replace("<p>", "").replace("</p>", "").replace("&nbsp;", "").replace("?", "");

// 把小说内容写入文件

writer.write(content + "\n");

}

 

}

 

System.out.println("第" + i + "章下载完成.........");

 

writer.write("\n\n");

writer.close();

reader.close();

} catch (Exception e) {

System.out.println("下载失败");

e.printStackTrace();

}

}

}

}

 

 

```java
package novelCrawler;

public class bqgThread {
	 
    public synchronized void  DownWait() {
    	try {
			this.wait();
		} catch (InterruptedException e) {
			// TODO 自动生成的 catch 块
			e.printStackTrace();
		}
    }
    public synchronized void  DownNotify() {
    	this.notify();;
    }
}

package ui;
import java.awt.*;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.ServerSocket;

import javax.swing.JFrame;
import javax.swing.JList;
import javax.swing.JPanel;
import javax.swing.JProgressBar;
import javax.swing.JTextArea;
import javax.swing.event.ChangeEvent;
import javax.swing.event.ChangeListener;

import novelCrawler.biquge;
public class crawlerUI implements ActionListener {
	TextField novelName = new TextField(10); 
	TextField novelSrc = new TextField("笔趣阁",10); 
	TextField start = new TextField(10); 
	JProgressBar jpb = new JProgressBar(); 
	 JPanel jp = new JPanel();
	 JProgressBar jpb2 = new JProgressBar(); 
	 JFrame myui = new JFrame("novelCrawler");
	 Button dl = new Button("开始下载");
	 
	public crawlerUI() { 
		
               myui.setSize(400,300);  
               myui.setLayout(null);
               Label label1=new Label("书  源");
               label1.setBounds(10, 15, 30, 20);
               
               myui.add(label1);
               
               novelSrc.setBounds(70,15, 90, 20);
               myui.add(novelSrc);
               
               Label label2=new Label("书  名");
               label2.setBounds(10, 45, 30, 20);
               myui.add(label2);
               
               novelName.setBounds(70,45, 90, 20);
               myui.add(novelName);
             
               Label label3=new Label("开始章节");
               label3.setBounds(10, 75, 60, 20);
               myui.add(label3);
               
               jpb2.setBounds(70, 150, 200, 30);
               myui.add(jpb2);
               start.setBounds(70, 75, 90, 20);
               myui.add(start);
               
               jpb.setBounds(70, 100, 200, 30);
               myui.add(jpb);
               
           
               
               jpb.setValue(1);
               jpb.setStringPainted(true);
              jpb.setVisible(false);
              jpb2.setVisible(false);
              jpb2.setStringPainted(true);
              
               
               dl.setBounds(150, 200, 70,40 );
               myui.add(dl);
               
          
               dl.addActionListener(this);
        myui.setVisible(true);               
       // myui.setResizable(true);
        
	}
	static int stop = 0;
	public void actionPerformed(ActionEvent ev) {
		String name = novelName.getText();
		System.out.println(name);
		String start1 = start.getText();
		System.out.println(start1);
		
		Thread bqg = new Thread(new Runnable() {
            @Override
            public void run() {
            	
                biquge bq = new biquge(name,start1,jpb,jpb2);
                
            }
           
        });
		
		
	
		if(stop==1) {
			dl.setLabel("继续");
			stop = 2;
			
			
			}
			else if(stop==2){
				dl.setLabel("暂停");
				stop = 1;
			
			}
			else {

				
				bqg.start();
				stop = 1;
				dl.setLabel("暂停");
			}
		
		
	}
	
	
}

package ui;
import javax.swing.*;
import javax.swing.JTextArea;

public class DownMsgUI {
		
	public JFrame dm ;
	public JTextArea ta ;
	public JScrollPane sp;
	public DownMsgUI(){
	
		dm	= new JFrame("下载信息");
		ta= new JTextArea();
	sp=new JScrollPane(ta);
	dm.setBounds(500, 500, 500, 500);
    ta.setBounds(70, 200, 200, 100);
    dm.add(sp);
    
    
    
    dm.setVisible(true);
	}
}