Question

我尝试使用python + selenium + phantomjs处理https请求，但无法从显示我adding certification is strongly advised的服务器收到响应，这是我的get_request函数。

def get_random_x_forwarded_for():
    # 得到随机x-forwarded-for值
    numbers = []
    while not numbers or numbers[0] in (10, 172, 192):
        numbers = random.sample(range(1, 255), 4)
    return '.'.join(str(_) for _ in numbers)

def get_random_ua():
    # 得到随机user-agent值
    import os
    if os.path.exists("dicts/user-agent.txt"):
        f = open(ModulePath + "dicts/user-agents.txt", "r+")
        all_user_agents = f.readlines()
        f.close()
    else:
        all_user_agents = [
            "Mozilla/4.0 (Mozilla/4.0; MSIE 7.0; Windows NT 5.1; FDM; SV1)",
            "Mozilla/4.0 (Mozilla/4.0; MSIE 7.0; Windows NT 5.1; FDM; SV1; .NET CLR 3.0.04506.30)",
            "Mozilla/4.0 (Windows; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)",
            "Mozilla/4.0 (Windows; U; Windows NT 5.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.33 Safari/532.0",
            "Mozilla/4.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.59 Safari/525.19",
            "Mozilla/4.0 (compatible; MSIE 6.0; Linux i686 ; en) Opera 9.70",
        ]
    random_ua_index = random.randint(0, len(all_user_agents) - 1)
    ua = re.sub(r"(\s)$", "", all_user_agents[random_ua_index])
    return ua

def get_request(url, by="MechanicalSoup", proxyUrl="", cookie=""):
    code = None
    title = None
    content = None
    if by == "seleniumPhantomJS":
        hasFormAction=False
        formActionValue=""
        from selenium import webdriver
        from selenium.common.exceptions import TimeoutException
        import time
        if proxyUrl == "" or proxyUrl == 0:
            service_args_value = ['--ignore-ssl-errors=true', '--ssl-protocol=any']
        if proxyUrl != "" and proxyUrl != 0:
            proxyType = proxyUrl.split(":")[0]
            proxyValueWithType = proxyUrl.split("/")[-1]
            service_args_value = ['--ignore-ssl-errors=true', '--ssl-protocol=any',
                                  '--proxy=%s' % proxyValueWithType, '--proxy-type=%s' % proxyType]
        # final_url=driver.current_url
        # print("正在访问的url是这个:\n"+final_url)
        # driver.quit()

        try:
            from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
            if cookie != "":
                dcap = dict(DesiredCapabilities.PHANTOMJS)
                dcap["phantomjs.page.settings.cookie"] = cookie
                dcap["phantomjs.page.settings.userAgent"] = get_random_ua()
                driver = webdriver.PhantomJS(service_args=service_args_value, desired_capabilities=dcap)
            else:
                driver = webdriver.PhantomJS(service_args=service_args_value)

            print(111111111)
            driver.implicitly_wait(5)
            driver.set_page_load_timeout(5)

            driver.get(url)
            print(222222222222222)
            # http://www.cnblogs.com/fnng/p/3269450.html
            originalCookie = driver.get_cookies()

            #print("current cookie is:\n" + str(originalCookie))

            import random
            code = 200 
            title = driver.title
            content = driver.page_source
            a=re.search(r'''(<.*type=('|")?submit('|")?.*>)''',content,re.I)
            if a:
                hasFormAction=True
                print(a.group(1))
                input(67666)
            else:
                pass

            print("len content is :\n" + str(len(content)))
            print("title is :\n" + title)

            if re.search(r"(页面不存在)|(未找到页面)|(page not found)|(404)",title+content,re.I):
                return get_request(url,by="MechanicalSoup")
            # time.sleep(5) # Let the user actually see something!
            # driver.quit()

        except TimeoutException as e:
            # Handle your exception here
            print(e)
        finally:
            driver.quit()

        return {
            'code': code,
            'title': title,
            'content': content,
            #True or False
            'hasFormAction':hasFormAction,
            #eg,https://www.baidu.com^a=1&b=2
            #eg,https://www.baidu.com/?a=1&b=2
            'formActionValue':formActionValue}

    else:
        import mechanicalsoup

        try:
            browser = mechanicalsoup.Browser(soup_config={"features": "lxml"})
            ua = get_random_ua()
            browser.session.headers.update({'User-Agent': '%s' % ua})
            # headers=browser.session.headers
            # if 'Cookie' in headers:
            # originalCookie=headers['Cookie']
            if cookie == "":
                pass
            else:
                browser.session.headers.update({'Cookie': '%s' % cookie})
            # print(originalCookie)
            x_forwarded_for = get_random_x_forwarded_for()
            browser.session.headers.update(
                {'X-Forwarded-For': '%s' % x_forwarded_for})


            result = browser.get(url,
                    timeout=10,verify=False)
            # print(dir(result))
            code = result.status_code
            content = result.content
            import chardet
            bytesEncoding = chardet.detect(content)['encoding']
            # print(bytesEncoding)
            content = content.decode(bytesEncoding)
            title = BeautifulSoup(content, "lxml").title
            if title is not None:
                title_value = title.string
            else:
                title_value = None
        except:

            code = 0
            title_value = "you may be blocked or the code doesn't handle ssl certificate well"
            content = 'can not get html content this time,may be blocked by the server to request'

        return_value = {
            'code': code,
            'title': title_value,
            'content': content}
        # print("访问当前url为:\n\t"+url+"\ntitle如下:")
        # print("\t"+str(return_value['title']))
        return return_value

b=get_request("https://www.zoomeye.org/search/advanced",by="seleniumPhantomJS")
#b=get_request("https://www.zoomeye.org/search/advanced",by="MechanicalSoup")
print(b)

我用的时候 b=get_request("https://www.zoomeye.org/search/advanced",by="seleniumPhantomJS")，它只会打印11111..1并且没有打印2222.2就会卡在那里，然后我尝试使用b=get_request("https://www.zoomeye.org/search/advanced",by="MechanicalSoup")，这次我得到以下错误调试信息：

/usr/local/lib/python3.6/site-packages/requests/packages/urllib3/connectionpool.py:852: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings InsecureRequestWarning)

我想使用python3 + selenium + phantomjs来处理这个adding certificate verification is strongly advised错误，有人可以帮帮我吗？

后来，我尝试直接使用没有硒的phantomjs：

phantomjs --ignore-ssl-errors=true --ssl-protocol=any --web-security=false 1.js 1.js具有以下内容：

var webPage = require('webpage');
var page = webPage.create();
console.log('test666');
page.open('https://www.zoomeye.org/search/advanced', function (status) {
  var content = page.content;
  console.log('Content: ' + content);
  phantom.exit();
});

然而，它不再起作用，可能这是phantomjs的错误:(

Answer 1

浏览以下网址。它将为您提供使用https与phantomjs所需的功能： -

http://phantomjs.org/api/command-line.html

您需要的主要功能如下： -

- web-security = [true | false]启用Web安全性并禁止跨域XHR（默认为true）。也接受：[是|否]。
- ssl-protocol = [sslv3 | sslv2 | tlsv1 | tlsv1.1 | tlsv1.2 | any']为安全连接设置SSL协议（默认为SSLv3）。并非所有价值观可能受支持，具体取决于系统OpenSSL库。
- ignore-ssl-errors = [true | false]忽略SSL错误，例如过期或自签名证书错误（默认为false）。也接受： [是|否]。

我有一个适用于我的java代码： -

private static Capabilities getPhantomCapabilities(String OS) {
    DesiredCapabilities capabilities = null;
    ArrayList<String> cliArgsCap = new ArrayList<String>();
    capabilities = DesiredCapabilities.phantomjs();
    cliArgsCap.add("--web-security=false");
    cliArgsCap.add("--ssl-protocol=any");
    cliArgsCap.add("--ignore-ssl-errors=true");
    capabilities.setCapability("takesScreenshot", true);
    capabilities.setJavascriptEnabled(true);
    capabilities.setCapability(
        PhantomJSDriverService.PHANTOMJS_CLI_ARGS, cliArgsCap);
    capabilities.setCapability(
        PhantomJSDriverService.PHANTOMJS_GHOSTDRIVER_CLI_ARGS,
            new String[] { "--logLevel=2" });
    return capabilities;
}

在python中转换代码。它应该工作

希望它会对你有所帮助:)。

Answer 2

兄弟你的https代理能用吗，我用http的代理可以使用但是用https的代理都用不了，用python请求使用https代理是没问题的，使用代码如下

service_args = [
    '--proxy='+ip[1],
    '--proxy-type=https',
]
driver = webdriver.PhantomJS(executable_path= os.getcwd()+'/driver/phantomjs.exe',service_args=service_args) #加载网址

python + selenium + phantomjs无法处理“强烈建议添加认证”的https网址

2 个答案: