最近弄爬虫，遇到的一个问题就是如何使用post方法模拟登陆爬取网页。下面是极简版的代码：


import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;

import java.net.HttpURLConnection;
import java.net.URL;

import java.util.HashMap;

public class test {

    //post请求地址
    private static final String POST_URL = "";
    
    //模拟谷歌浏览器请求
    private static final String USER_AGENT = "";
    
    //用账号登录某网站后 请求POST_URL链接获取cookie
    private static final String COOKIE = "";
    
    //用账号登录某网站后 请求POST_URL链接获取数据包
    private static final String REQUEST_DATA =  "";
    
    public static void main(String[] args) throws Exception {
        HashMap<String, String> map = postCapture(REQUEST_DATA);
        String responseCode = map.get("responseCode");
        String value = map.get("value");
        
        while(!responseCode.equals("200")){
            map =  postCapture(REQUEST_DATA);
            responseCode = map.get("responseCode");
            value = map.get("value");
        }
        
        //打印爬取结果
        System.out.println(value);
    }
    
    private static HashMap<String, String> postCapture(String requestData) throws Exception{
        HashMap<String, String> map = new HashMap<>();
        
        URL url = new URL(POST_URL);
        HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
        httpConn.setDoInput(true); // 设置输入流采用字节流
        httpConn.setDoOutput(true); // 设置输出流采用字节流
        httpConn.setUseCaches(false); //设置缓存
        httpConn.setRequestMethod("POST");//POST请求
        httpConn.setRequestProperty("User-Agent", USER_AGENT);
        httpConn.setRequestProperty("Cookie", COOKIE);
        
        PrintWriter out = new PrintWriter(new OutputStreamWriter(httpConn.getOutputStream(), "UTF-8"));
        out.println(requestData);
        out.close();

        int responseCode = httpConn.getResponseCode();
        StringBuffer buffer = new StringBuffer();
        if (responseCode == 200) {
            BufferedReader reader = new BufferedReader(new InputStreamReader(httpConn.getInputStream(), "UTF-8"));
            String line = null;
            while ((line = reader.readLine()) != null) {
                buffer.append(line);
            }
            reader.close();
            httpConn.disconnect();
        }
        
        map.put("responseCode", new Integer(responseCode).toString());
        map.put("value", buffer.toString());
        return map;
    }

}

原文地址：
http://wangxin123.com/2016/12/19/%E4%BD%BF%E7%94%A8Post%E6%96%B9%E6%B3%95%E6%A8%A1%E6%8B%9F%E7%99%BB%E9%99%86%E7%88%AC%E5%8F%96%E7%BD%91%E9%A1%B5/

转载于:https://www.cnblogs.com/wangxin37/p/6398743.html

使用Post方法模拟登陆爬取网页
 


最近弄爬虫，遇到的一个问题就是如何使用post方法模拟登陆爬取网页。下面是极简版的代码：



import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;

import java.net.HttpURLConnection;
import java.net.URL;

import java.util.HashMap;

public class test {

    //post请求地址
    private static final String POST_URL = "";
    
    //模拟谷歌浏览器请求
    private static final String USER_AGENT = "";
    
    //用账号登录某网站后 请求POST_URL链接获取cookie
    private static final String COOKIE = "";
    
    //用账号登录某网站后 请求POST_URL链接获取数据包
    private static final String REQUEST_DATA =  "";
    
    public static void main(String[] args) throws Exception {
        HashMap<String, String> map = postCapture(REQUEST_DATA);
        String responseCode = map.get("responseCode");
        String value = map.get("value");
        
        while(!responseCode.equals("200")){
            map =  postCapture(REQUEST_DATA);
            responseCode = map.get("responseCode");
            value = map.get("value");
        }
        
        //打印爬取结果
        System.out.println(value);
    }
    
    private static HashMap<String, String> postCapture(String requestData) throws Exception{
        HashMap<String, String> map = new HashMap<>();
        
        URL url = new URL(POST_URL);
        HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
        httpConn.setDoInput(true); // 设置输入流采用字节流
        httpConn.setDoOutput(true); // 设置输出流采用字节流
        httpConn.setUseCaches(false); //设置缓存
        httpConn.setRequestMethod("POST");//POST请求
        httpConn.setRequestProperty("User-Agent", USER_AGENT);
        httpConn.setRequestProperty("Cookie", COOKIE);
        
        PrintWriter out = new PrintWriter(new OutputStreamWriter(httpConn.getOutputStream(), "UTF-8"));
        out.println(requestData);
        out.close();

        int responseCode = httpConn.getResponseCode();
        StringBuffer buffer = new StringBuffer();
        if (responseCode == 200) {
            BufferedReader reader = new BufferedReader(new InputStreamReader(httpConn.getInputStream(), "UTF-8"));
            String line = null;
            while ((line = reader.readLine()) != null) {
                buffer.append(line);
            }
            reader.close();
            httpConn.disconnect();
        }
        
        map.put("responseCode", new Integer(responseCode).toString());
        map.put("value", buffer.toString());
        return map;
    }

}

原文地址：
http://wangxin123.com/2016/12/19/%E4%BD%BF%E7%94%A8Post%E6%96%B9%E6%B3%95%E6%A8%A1%E6%8B%9F%E7%99%BB%E9%99%86%E7%88%AC%E5%8F%96%E7%BD%91%E9%A1%B5/




转载于:https://www.cnblogs.com/panxuejun/p/7476222.html

今天和京东斗智斗勇了一天，终于还是成功了，记录下过程
 
验证码图片的处理
验证码原图的匹配
最最最最恶心的 滑动轨迹
代码

验证码图片的处理
京东的验证码的完整图实际上只有10张(2019.01.26)，我直接把10张图给截出来了，由于PS不过关，我是选的缺口尽量靠右下方的图，然后把滑块移动到缺口处截图。方法基本笨，但基本能行。

效果图如下


验证码原图的匹配
现在手头有10张验证码原图，在登陆验证时，需要把待滑动的图与原图匹配起来，如需要将下面的带缺口的图与上面的图匹配。



这里参考了Pillow实现图片对比
def img_difference(img1, img2):
    """
    比较两图片差异
    :param img1: 图片1
    :param img2: 图片2名称
    :return: 差异大小
    """
    image1 = img1
    image2 = Image.open(img2)
    h1 = image1.histogram()
    h2 = image2.histogram()
    result = math.sqrt(reduce(operator.add, list(map(lambda a, b: (a - b) ** 2, h1, h2))) / len(h1))
    return result

def img_match(image):
    """
    比较两图片差异
    :param image: 带缺口图片 
    :return: 匹配到的图片名称
    """
    image_list = []
    for i in range(1, 11):
        image_list.append('jd' + str(i) + '.png') # 我将图片命名为jd1.png,jd2.png,…，jd10.png
    results = {}
    for img in image_list:
        results[img] = self.img_difference(image, img)
    image_match = min(results, key=lambda x: results[x])
    print(image_match)
    return image_match

大概就是先把图像转为影像直方图，计算两张图片亮度累积的像素数量的均方误差，均方误差最小的即为匹配结果。
或者直接一个函数
def img_match(self, image):
    """
    比较两图片差异
    :param image: 带缺口图片
    :return: 匹配到的图片名称
    """
    image_list = []
    for i in range(1, 11):
        image_list.append('jd' + str(i) + '.png')
    results = {}
    for img in image_list:
        results[img] = self.img_difference(image, img)
    image_match = min(results, key=lambda x: results[x])
    print(image_match)
    return image_match

最最最最恶心的 滑动轨迹
尝试过各种轨迹，基本都失败了，想说不然查一查别人怎么做的，发现也现在(2019.01.26)也都不能用，自己人工拖了几下，发现，把验证码拉过头停顿一会儿再往回拉，竟然都能成功！
def get_track(distance):
    """
    根据偏移量获取移动轨迹
    这里把距离加长，拉过头
    :param distance: 偏移量
    :return: 移动轨迹
    """
    # 加长距离，拉过头
    distance += 6
    # 移动轨迹
    track = []
    # 当前位移
    current = 0
    # 变速阈值
    if distance > 100:
        mid1 = distance * 1 / 3
        mid2 = distance * 7 / 8
    else:
        mid1 = distance * 1 / 2
        mid2 = distance * 5 / 6
    # 间隔时间
    t = 0.3
    # 初速度
    v = 3
    while current < distance:
        if current < mid1:
            a = random.uniform(3, 5)
        elif current < mid2:
            a = random.uniform(0, 1)
        else:
            a = - random.uniform(9, 12)
        v0 = v
        v = v0 + a * t
        move = v0 * t + 1 / 2 * a * t * t
        current += move
        track.append(round(move))
    return track

def move_to_gap(knob, track):
    """
    拖动滑块到缺口处
    :param knob: 圆球
    :param track: 轨迹
    :return:
    """
    ActionChains(browser).click_and_hold(knob).perform()
    for x in track:
        ActionChains(browser).move_by_offset(xoffset=x, yoffset=0).perform()
    # 拉过头后停留一会，再往回拉
    time.sleep(0.5)
    for x in [-1, -2, -2, -1]:
        ActionChains(browser).move_by_offset(xoffset=x, yoffset=0).perform()
    time.sleep(0.3)
    ActionChains(browser).release().perform()

代码
完整代码如下
import math
import random
import time
from functools import reduce
import operator
from io import BytesIO
from PIL import Image
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC



USER_NAME = ''  # 账户
PASSWORD = ''  # 密码
MULTIPE = 1.5  # 显示比例，我这里是150%


class CrackBili(object):
    def __init__(self):
        self.url = 'https://www.jd.com'
        self.browser = webdriver.Chrome()
        self.browser.maximize_window()
        self.wait = WebDriverWait(self.browser, 10)
        self.wait_pass = WebDriverWait(self.browser, 1)
        self.user_name = USER_NAME
        self.password = PASSWORD

    def __del__(self):
        self.browser.close()

    def open(self):
        """
        打开网页，并切换到登陆界面,输入账号密码
        :return:
        """
        self.browser.get(self.url)
        link_login = self.wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'link-login')))
        link_login.click()
        login_tab = self.wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'login-tab-r')))
        login_tab.click()
        user_name = self.wait.until(EC.presence_of_element_located((By.ID, 'loginname')))
        password = self.wait.until(EC.presence_of_element_located((By.ID, 'nloginpwd')))
        user_name.send_keys(self.user_name)
        password.send_keys(self.password)
        login_button = self.wait.until(EC.element_to_be_clickable((By.ID, 'loginsubmit')))
        login_button.click()

    def get_knob(self):
        """
        获取圆球对象
        :return: 圆球对象
        """
        knob = self.wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,
            '#JDJRV-wrap-loginsubmit > div > div > div > div.JDJRV-slide-bg > div.JDJRV-slide-inner.JDJRV-slide-btn')))
        return knob

    def get_position(self):
        """
        获取验证码位置
        :return: 验证码位置元组
        """
        img = self.wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'JDJRV-bigimg')))
        location = img.location
        size = img.size
        top, bottom, left, right = location['y'], location['y'] + size['height'], location['x'], location['x'] + size[
            'width']
        return (top, bottom, left, right)

    def get_screenshot(self):
        """
        获取网页截图
        :return: 截图对象
        """
        screenshot = self.browser.get_screenshot_as_png()
        screenshot = Image.open(BytesIO(screenshot))
        return screenshot

    def get_bili_image(self, name='captcha.png'):
        """
        获取验证码图片
        :return: 图片对象
        """
        top, bottom, left, right = self.get_position()
        print('验证码位置', left, top, right, bottom)
        screenshot = self.get_screenshot()
        captcha = screenshot.crop(map(lambda x: x * MULTIPE, (left, top, right, bottom)))
        captcha.save(name)
        return captcha

    def img_difference(self, img1, img2):
        """
        比较两图片差异
        :param img1: 图片1
        :param img2: 图片2名称
        :return: 差异大小
        """
        image1 = img1
        image2 = Image.open(img2)
        h1 = image1.histogram()
        h2 = image2.histogram()
        result = math.sqrt(reduce(operator.add, list(map(lambda a, b: (a - b) ** 2, h1, h2))) / len(h1))
        return result

    def img_match(self, image):
        """
        比较两图片差异
        :param image: 带缺口图片
        :return: 匹配到的图片名称
        """
        image_list = []
        for i in range(1, 11):
            image_list.append('jd' + str(i) + '.png')
        results = {}
        for img in image_list:
            results[img] = self.img_difference(image, img)
        image_match = min(results, key=lambda x: results[x])
        print(image_match)
        return image_match

    def is_pixel_equal(self, image1, image2, x, y):
        """
        判断两个像素是否相同
        :param image1: 图片1
        :param image2: 图片2
        :param x: 位置x
        :param y: 位置y
        :return: 像素是否相同
        """
        # 取两个图片的像素点
        pixel1 = image1.load()[x, y]
        pixel2 = image2.load()[x, y]
        threshold = 60
        if abs(pixel1[0] - pixel2[0]) < threshold and abs(pixel1[1] - pixel2[1]) < threshold and abs(
                pixel1[2] - pixel2[2]) < threshold:
            return True
        else:
            return False
    def get_gap(self, image1, image2):
        """
        获取缺口偏移量
        :param image1: 不带缺口图片
        :param image2: 带缺口图片
        :return:
        """
        left = round(38.6 * MULTIPE) + 2  # 滑块最右测在验证码图片上的x坐标为38.6
        for i in range(left, image1.size[0]):
            for j in range(image1.size[1]):
                if not self.is_pixel_equal(image1, image2, i, j):
                    left = i
                    return left
        return left

    def get_track(self, distance):
        """
        根据偏移量获取移动轨迹
        这里把距离加长，拉过头
        :param distance: 偏移量
        :return: 移动轨迹
        """
        # 加长距离，拉过头
        distance += 6
        # 移动轨迹
        track = []
        # 当前位移
        current = 0
        # 变速阈值
        if distance > 100:
            mid1 = distance * 1 / 3
            mid2 = distance * 7 / 8
        else:
            mid1 = distance * 1 / 2
            mid2 = distance * 5 / 6
        # 间隔时间
        t = 0.3
        # 初速度
        v = 3
        while current < distance:
            if current < mid1:
                a = random.uniform(3, 5)
            elif current < mid2:
                a = random.uniform(0, 1)
            else:
                a = - random.uniform(9, 12)
            v0 = v
            v = v0 + a * t
            move = v0 * t + 1 / 2 * a * t * t
            current += move
            track.append(round(move))
        return track

    def move_to_gap(self, knob, track):
        """
        拖动滑块到缺口处
        :param knob: 圆球
        :param track: 轨迹
        :return:
        """
        ActionChains(self.browser).click_and_hold(knob).perform()
        for x in track:
            ActionChains(self.browser).move_by_offset(xoffset=x, yoffset=0).perform()
        # 拉过头后停留一会，再往回拉
        time.sleep(0.5)
        for x in [-1, -2, -2, -1]:
            ActionChains(self.browser).move_by_offset(xoffset=x, yoffset=0).perform()
        time.sleep(0.3)
        ActionChains(self.browser).release().perform()

    def crack(self):
        # 获取验证码图片
        image = self.get_bili_image('captcha.png')
        # 匹配图片
        image_match = Image.open(self.img_match(image))
        # 获取缺口位置
        gap = self.get_gap(image_match, image)
        print('缺口位置', gap)
        # 截图中是150%的距离，要除掉
        gap = int(gap / MULTIPE)
        # 获取移动轨迹
        track = self.get_track(gap)
        knob = self.get_knob()
        # 拖动滑块
        self.move_to_gap(knob, track)
        try:
            success = self.wait_pass.until(
                EC.text_to_be_present_in_element((By.CLASS_NAME, 'JDJRV-slide-bar-center'), '拼接成功'))
            print(success)
        except TimeoutException:
            success = None

        #失败后重试
        if not success:
            self.crack()
        else:
            time.sleep(10)
            self.__del__()


if __name__ == '__main__':
    crack = CrackBili()
    # 打开网页，并切换到登陆界面, 输入账号密码
    crack.open()
    # 拉动滑块直至通过验证
    crack.crack()


图片后续会上传至GitHub

分辨率 1920*1080  显示比例150%，其他显示比例可能会有所有不同，可以自行截图
方法原创，至今看到的方法都不能通过验证，当然我这个可能过一段时间也通过不了了。
转载请注明，欢迎交流

Scrapy模拟表单和Ajax发送POST请求
scrapy.FormRequest
普通请求使用scrapy.Request类就可以实现，但是遇到模拟表单或Ajax提交post请求的时候，Request类就不如 子类 FormRequest类方便了，因为他自带 formdata ，专门用来设置表单字段数据，默认method也是POST。
def start_requests(self):
    form_data = {'f1':'1', 'f2':'100'}  # 表单数据，字典格式，注意数字也要用引号引起来，否则报错。
    yield scrapy.FormRequest(url, formdata=form_data) # 还可以通过callback修改回调函数等
1
2
3
参考: 
http://scrapy-chs.readthedocs.io/zh_CN/0.24/topics/request-response.html?highlight=post#formrequest-objects 
https://stackoverflow.com/questions/39012902/scrapy-making-request-with-post-method
scrapy.http.FormRequest
还有其他方法，如scrapy.http.FormRequest，但是感觉不如以上方法方便：

return [scrapy.http.FormRequest(
    self.myurl, 
    formdata={'f1':'123','f2':'456'},
    callback=self.parse)]
eg:
 from scrapy.item import Item, Field
from scrapy.http import FormRequest
from scrapy.spider import BaseSpider


class DeltaItem(Item):
    title = Field()
    link = Field()
    desc = Field()


class DmozSpider(BaseSpider):
    name = "delta"
    allowed_domains = ["delta.com"]
    start_urls = ["http://www.delta.com"]

    def parse(self, response):
        yield FormRequest.from_response(response,
                                        formname='flightSearchForm',
                                        formdata={'departureCity[0]': 'JFK',
                                                  'destinationCity[0]': 'SFO',
                                                  'departureDate[0]': '07.20.2013',
                                                  'departureDate[1]': '07.28.2013'},
                                        callback=self.parse1)

    def parse1(self, response):
        print response.status
scrapy基础知识之发送POST请求：
可以使用 yield scrapy.FormRequest(url, formdata, callback)方法发送POST请求。
如果希望程序执行一开始就发送POST请求，可以重写Spider类的start_requests(self) 方法，并且不再调用start_urls里的url。
class mySpider(scrapy.Spider):
    # start_urls = ["http://www.example.com/"]

    def start_requests(self):
        url = 'http://www.renren.com/PLogin.do'

        # FormRequest 是Scrapy发送POST请求的方法
        yield scrapy.FormRequest(
            url = url,
            formdata = {"email" : "xxx", "password" : "xxxxx"},
            callback = self.parse_page
        )
    def parse_page(self, response):
        # do something

scrapy基础知识之 使用FormRequest.from_response()方法模拟用户登录：
通常网站通过 实现对某些表单字段（如数据或是登录界面中的认证令牌等）的预填充
使用Scrapy抓取网页时，如果想要预填充或重写像用户名、用户密码这些表单字段， 可以使用 FormRequest.from_response() 方法实现。
下面是使用这种方法的爬虫例子:
import scrapy

class LoginSpider(scrapy.Spider):
    name = 'example.com'
    start_urls = ['http://www.example.com/users/login.php']

    def parse(self, response):
        return scrapy.FormRequest.from_response(
            response,
            formdata={'username': 'john', 'password': 'secret'},
            callback=self.after_login
        )

    def after_login(self, response):
        # check login succeed before going on
        if "authentication failed" in response.body:
            self.log("Login failed", level=log.ERROR)
            return