我写了一个简单的PhantomJS脚本,可以自动化网站上的流程。此过程的一部分是提交表单。几个月来,我一直在使用这个简单的程序,使用高质量的代理提供商。最近我换了供应商,现在我无法绕过这个表格。在进一步测试(在本地计算机上设置代理)后,我发现使用代理作为socks而不是http允许我根据需要绕过表单。
但是,在尝试设置“--proxy-type = socks5”时出现错误:
Network - Resource request error: QNetworkReply::NetworkError(UnknownNetworkError) ( "SOCKS version 5 protocol error" ) URL: "https://api.ipify.org/?format=json%27"
我尝试运行的命令行是:
phantomjs --debug --proxy-type=socks5 --proxy=69.46.80.98:16151 ip.js
ip.js的内容:
var fs = require('fs');
var system = require('system');
var page = new WebPage(), testindex = 0, loadInProgress = false;
page.setReferrer = false;
var userAgents = [
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
'Mozilla/5.0 (Windows NT 5.1; rv:52.0) Gecko/20100101 Firefox/52.0'
];
var sessionAgent = userAgents[Math.floor(Math.random() * userAgents.length)];
page.settings.userAgent = sessionAgent;
page.settings.resourceTimeout = 600000;
page.settings.loadImages = false;
page.onConsoleMessage = function(msg) {
console.log(msg);
};
page.onLoadStarted = function() {
loadInProgress = true;
console.log("load started");
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log("load finished");
};
var steps = [
function() {
page.open('https://api.ipify.org/?format=json%27');
},
function() {
console.log( page.plainText );
}
];
interval = setInterval(function() {
if (!loadInProgress && typeof steps[testindex] == "function") {
console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}, 50);
完整的调试输出:
root@earth:~# phantomjs --debug=true --proxy-type=socks5 --proxy=69.46.80.98:16151 /home/phantomjs/downloaders/ip.js
2017-12-17T10:48:36 [DEBUG] CookieJar - Created but will not store cookies (use option '--cookies-file=<filename>' to enable persistent cookie storage)
2017-12-17T10:48:36 [DEBUG] Set "socks5" proxy to: "69.46.80.98" : 16151
2017-12-17T10:48:36 [DEBUG] Phantom - execute: Configuration
2017-12-17T10:48:36 [DEBUG] 0 objectName : ""
2017-12-17T10:48:36 [DEBUG] 1 cookiesFile : ""
2017-12-17T10:48:36 [DEBUG] 2 diskCacheEnabled : "false"
2017-12-17T10:48:36 [DEBUG] 3 maxDiskCacheSize : "-1"
2017-12-17T10:48:36 [DEBUG] 4 diskCachePath : ""
2017-12-17T10:48:36 [DEBUG] 5 ignoreSslErrors : "false"
2017-12-17T10:48:36 [DEBUG] 6 localUrlAccessEnabled : "true"
2017-12-17T10:48:36 [DEBUG] 7 localToRemoteUrlAccessEnabled : "false"
2017-12-17T10:48:36 [DEBUG] 8 outputEncoding : "UTF-8"
2017-12-17T10:48:36 [DEBUG] 9 proxyType : "socks5"
2017-12-17T10:48:36 [DEBUG] 10 proxy : "69.46.80.98:16151"
2017-12-17T10:48:36 [DEBUG] 11 proxyAuth : ":"
2017-12-17T10:48:36 [DEBUG] 12 scriptEncoding : "UTF-8"
2017-12-17T10:48:36 [DEBUG] 13 webSecurityEnabled : "true"
2017-12-17T10:48:36 [DEBUG] 14 offlineStoragePath : ""
2017-12-17T10:48:36 [DEBUG] 15 localStoragePath : ""
2017-12-17T10:48:36 [DEBUG] 16 localStorageDefaultQuota : "-1"
2017-12-17T10:48:36 [DEBUG] 17 offlineStorageDefaultQuota : "-1"
2017-12-17T10:48:36 [DEBUG] 18 printDebugMessages : "true"
2017-12-17T10:48:36 [DEBUG] 19 javascriptCanOpenWindows : "true"
2017-12-17T10:48:36 [DEBUG] 20 javascriptCanCloseWindows : "true"
2017-12-17T10:48:36 [DEBUG] 21 sslProtocol : "default"
2017-12-17T10:48:36 [DEBUG] 22 sslCiphers : "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-RC4-SHA:ECDHE-RSA-RC4-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:RC4-SHA:RC4-MD5"
2017-12-17T10:48:36 [DEBUG] 23 sslCertificatesPath : ""
2017-12-17T10:48:36 [DEBUG] 24 sslClientCertificateFile : ""
2017-12-17T10:48:36 [DEBUG] 25 sslClientKeyFile : ""
2017-12-17T10:48:36 [DEBUG] 26 sslClientKeyPassphrase : ""
2017-12-17T10:48:36 [DEBUG] 27 webdriver : ":"
2017-12-17T10:48:36 [DEBUG] 28 webdriverLogFile : ""
2017-12-17T10:48:36 [DEBUG] 29 webdriverLogLevel : "INFO"
2017-12-17T10:48:36 [DEBUG] 30 webdriverSeleniumGridHub : ""
2017-12-17T10:48:36 [DEBUG] Phantom - execute: Script & Arguments
2017-12-17T10:48:36 [DEBUG] script: "/home/phantomjs/downloaders/ip.js"
2017-12-17T10:48:36 [DEBUG] Phantom - execute: Starting normal mode
2017-12-17T10:48:36 [DEBUG] WebPage - setupFrame ""
2017-12-17T10:48:36 [DEBUG] FileSystem - _open: ":/modules/fs.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:36 [DEBUG] FileSystem - _open: ":/modules/system.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:36 [DEBUG] FileSystem - _open: ":/modules/webpage.js" QMap(("mode", QVariant(QString, "r")))
step 1
2017-12-17T10:48:36 [DEBUG] WebPage - updateLoadingProgress: 10
load started
2017-12-17T10:48:37 [DEBUG] skipping hostname of len 13
2017-12-17T10:48:37 [DEBUG] Network - Resource request error: QNetworkReply::NetworkError(UnknownNetworkError) ( "SOCKS version 5 protocol error" ) URL: "https://api.ipify.org/?format=json%27"
2017-12-17T10:48:37 [DEBUG] WebPage - updateLoadingProgress: 100
load finished
2017-12-17T10:48:37 [DEBUG] WebPage - setupFrame ""
2017-12-17T10:48:37 [DEBUG] WebPage - setupFrame ""
step 2
test complete!
2017-12-17T10:48:37 [DEBUG] WebPage - setupFrame ""
2017-12-17T10:48:37 [DEBUG] WebPage - updateLoadingProgress: 10
2017-12-17T10:48:37 [DEBUG] WebPage - setupFrame ""
2017-12-17T10:48:37 [DEBUG] WebPage - updateLoadingProgress: 100
2017-12-17T10:48:37 [DEBUG] WebPage - setupFrame ""
2017-12-17T10:48:37 [DEBUG] FileSystem - _open: ":/modules/fs.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:37 [DEBUG] FileSystem - _open: ":/modules/system.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:37 [DEBUG] FileSystem - _open: ":/modules/webpage.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:37 [DEBUG] WebPage - updateLoadingProgress: 10
2017-12-17T10:48:37 [DEBUG] WebPage - setupFrame ""
2017-12-17T10:48:37 [DEBUG] FileSystem - _open: ":/modules/fs.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:37 [DEBUG] FileSystem - _open: ":/modules/system.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:37 [DEBUG] FileSystem - _open: ":/modules/webpage.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:37 [DEBUG] WebPage - updateLoadingProgress: 100
答案 0 :(得分:0)
If proxy is working this could be due to SSL errors. Try using these switches in command-line:
--ssl-protocol=any --ignore-ssl-errors