希望专家可以帮助我解决 Selenium/Cloudflare 之谜。我可以让网站在正常(非无头)Selenium 中加载,但无论我尝试什么,我都无法在无头中加载它。
我遵循了 StackOverflow 帖子(如 Is there a version of Selenium WebDriver that is not detectable?)中的建议。我还查看了 window
和 window.navigator
对象的所有属性,并修复了无头和非无头之间的所有差异,但仍以某种方式检测到无头。在这一点上,我非常好奇 Cloudflare 是如何找出差异的。谢谢你的时间!
我尝试过的事情列表:
cdc_
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument('--disable-blink-features=AutomationControlled')
(这是让网站以非无头方式加载所必需的)navigator.webdriver = undefined
navigator.plugins
、navigator.languages
和 navigator.mimeTypes
window.ScreenY
、window.screenTop
、window.outerWidth
、window.outerHeight
设置为非零值window.chrome
和 window.navigator.chrome
Modernizr
重复实验
为了让网站在正常(非无头)Selenium 中加载,您必须点击来自另一个网站的 _blank
链接(以便目标网站在另一个选项卡中打开)。要复制实验,请先创建一个内容为 <a href="https://poocoin.app" target="_blank">link</a>
的 html 文件,然后将这个 html 文件的路径粘贴到以下代码中。
以下版本(非无头)运行良好并加载网站,但如果您设置 options.headless = True
,它将卡在 Cloudflare 上。
from selenium import webdriver
import time
# Replace this with the path to your html file
FULL_PATH_TO_HTML_FILE = 'file:///Users/simplepineapple/html/url_page.html'
def visit_website(browser):
browser.get(FULL_PATH_TO_HTML_FILE)
time.sleep(3)
links = browser.find_elements_by_xpath("//a[@href]")
links[0].click()
time.sleep(10)
# Switch webdriver focus to new tab so that we can extract html
tab_names = browser.window_handles
if len(tab_names) > 1:
browser.switch_to.window(tab_names[1])
time.sleep(1)
html = browser.page_source
print(html)
print()
print()
if 'Charts' in html:
print('Success')
else:
print('Fail')
time.sleep(10)
options = webdriver.ChromeOptions()
# If options.headless = True, the website will not load
options.headless = False
options.add_argument("--window-size=1920,1080")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_argument('user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36')
browser = webdriver.Chrome(options = options)
browser.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
"source": '''
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
});
Object.defineProperty(navigator, 'plugins', {
get: function() { return {"0":{"0":{}},"1":{"0":{}},"2":{"0":{},"1":{}}}; }
});
Object.defineProperty(navigator, 'languages', {
get: () => ["en-US", "en"]
});
Object.defineProperty(navigator, 'mimeTypes', {
get: function() { return {"0":{},"1":{},"2":{},"3":{}}; }
});
window.screenY=23;
window.screenTop=23;
window.outerWidth=1337;
window.outerHeight=825;
window.chrome =
{
app: {
isInstalled: false,
},
webstore: {
onInstallStageChanged: {},
onDownloadProgress: {},
},
runtime: {
PlatformOs: {
MAC: 'mac',
WIN: 'win',
ANDROID: 'android',
CROS: 'cros',
LINUX: 'linux',
OPENBSD: 'openbsd',
},
PlatformArch: {
ARM: 'arm',
X86_32: 'x86-32',
X86_64: 'x86-64',
},
PlatformNaclArch: {
ARM: 'arm',
X86_32: 'x86-32',
X86_64: 'x86-64',
},
RequestUpdateCheckStatus: {
THROTTLED: 'throttled',
NO_UPDATE: 'no_update',
UPDATE_AVAILABLE: 'update_available',
},
OnInstalledReason: {
INSTALL: 'install',
UPDATE: 'update',
CHROME_UPDATE: 'chrome_update',
SHARED_MODULE_UPDATE: 'shared_module_update',
},
OnRestartRequiredReason: {
APP_UPDATE: 'app_update',
OS_UPDATE: 'os_update',
PERIODIC: 'periodic',
},
},
};
window.navigator.chrome =
{
app: {
isInstalled: false,
},
webstore: {
onInstallStageChanged: {},
onDownloadProgress: {},
},
runtime: {
PlatformOs: {
MAC: 'mac',
WIN: 'win',
ANDROID: 'android',
CROS: 'cros',
LINUX: 'linux',
OPENBSD: 'openbsd',
},
PlatformArch: {
ARM: 'arm',
X86_32: 'x86-32',
X86_64: 'x86-64',
},
PlatformNaclArch: {
ARM: 'arm',
X86_32: 'x86-32',
X86_64: 'x86-64',
},
RequestUpdateCheckStatus: {
THROTTLED: 'throttled',
NO_UPDATE: 'no_update',
UPDATE_AVAILABLE: 'update_available',
},
OnInstalledReason: {
INSTALL: 'install',
UPDATE: 'update',
CHROME_UPDATE: 'chrome_update',
SHARED_MODULE_UPDATE: 'shared_module_update',
},
OnRestartRequiredReason: {
APP_UPDATE: 'app_update',
OS_UPDATE: 'os_update',
PERIODIC: 'periodic',
},
},
};
['height', 'width'].forEach(property => {
const imageDescriptor = Object.getOwnPropertyDescriptor(HTMLImageElement.prototype, property);
// redefine the property with a patched descriptor
Object.defineProperty(HTMLImageElement.prototype, property, {
...imageDescriptor,
get: function() {
// return an arbitrary non-zero dimension if the image failed to load
if (this.complete && this.naturalHeight == 0) {
return 20;
}
return imageDescriptor.get.apply(this);
},
});
});
const getParameter = WebGLRenderingContext.getParameter;
WebGLRenderingContext.prototype.getParameter = function(parameter) {
if (parameter === 37445) {
return 'Intel Open Source Technology Center';
}
if (parameter === 37446) {
return 'Mesa DRI Intel(R) Ivybridge Mobile ';
}
return getParameter(parameter);
};
const elementDescriptor = Object.getOwnPropertyDescriptor(HTMLElement.prototype, 'offsetHeight');
Object.defineProperty(HTMLDivElement.prototype, 'offsetHeight', {
...elementDescriptor,
get: function() {
if (this.id === 'modernizr') {
return 1;
}
return elementDescriptor.get.apply(this);
},
});
'''
})
visit_website(browser)
browser.quit()