我正在尝试使用Python的请求模块抓取网站
这是一个我们只能在公司内部访问的网站。
每次我尝试使用get()抓取网站时，它就是这个
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>401 Authorization Required</title>
</head><body>
<h1>Authorization Required</h1>
<p>This server could not verify that you
are authorized to access the document
requested. Either you supplied the wrong
credentials (e.g., bad password), or your
browser doesn't understand how to supply
the credentials required.</p>
<hr>
<address>Apache/2.2.13 (Red Hat) Server at tcfabcert Port 80</address>
</body></html>
但是当我插入auth=('user','pass')时，它得到了200的响应，但是它并没有登录，并且每次我尝试使用post()方法登录时只是一直将我重定向到同一登录页面。
这是我的代码：
import requests
payload = {'userID': 'user',
'password': 'mypass',
'userLocale': 'US CA',
'userBU': 'MB',
'servName': 'Certification',
'login': 'Login',
'rememberFlag': 'false',
'hiddenHost': 'cpc44743'}
cookie = {'Cookie': 'JSESSIONID=AF1A7CC747492C3D4259B230A5F7635A.PKYCUbG8Dba7VqoxAzD4WA'}
header = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate',
'Accept-Language':'en-US,en;q=0.9',
'Cache-Control':'max-age=0',
'Connection':'keep-alive',
'Content-Length':'138',
'Content-Type':'application/x-www-form-urlencoded',
'Cookie':'JSESSIONID=AF1A7CC747492C3D4259B230A5F7635A.PKYCUbG8Dba7VqoxAzD4WA',
'Host':'tcfabcert',
'Origin':'http://tcfabcert',
'Referer':'http://tcfabcert/ws/ess/default.jsp',
'Upgrade-Insecure-Requests':'1',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'}
with requests.Session() as r:
url = "http://tcfabcert/ws/ess/"
res = r.get(url, auth=('user', 'password')).text
res2 = r.post(url, auth=('user', 'password'), datas=payload, headers=header, allow_redirects=True)
这是登录页面的代码
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
<html>
<head>
<meta name="GENERATOR"
content="IBM WebSphere Page Designer V3.5.2 for Windows" />
<meta http-equiv="Content-Style-Type" content="text/css" />
<title>Editorial System</title>
<link rel="stylesheet" type="text/css" href="/ws/ess/theme/Style2.css" />
<script type="text/javascript">
var GB_ROOT_DIR = "/ws/ess/greybox/";
</script>
<script type="text/javascript" src="/ws/ess/greybox/AJS.js"></script>
<script type="text/javascript" src="/ws/ess/greybox/AJS_fx.js"></script>
<script type="text/javascript" src="/ws/ess/greybox/gb_scripts.js"></script>
<link href="/ws/ess/greybox/gb_styles.css" rel="stylesheet" type="text/css" media="all" />
<script language="JavaScript1.2">
<!-- Hide from older browsers
// Original JavaScript code by Duncan Crombie: dcrombie at chirp.com.au
// Please acknowledge use of this code by including this header.
function getCookie(name) { // use: getCookie("name");
var re = new RegExp(name + "=([^;]+)");
var value = re.exec(document.cookie);
return (value != null) ? unescape(value[1]) : null;
}
var today = new Date();
var expiry = new Date(today.getTime() + 28 * 24 * 60 * 60 * 1000); // plus 28 days
function setCookie(name, value) { // use: setCookie("name", value);
document.cookie=name + "=" + escape(value) + "; expires=" + expiry.toGMTString();
}
// Stop hiding -->
function dispHandle(obj) {
if (obj.style.display == "none")
obj.style.display = "";
else
obj.style.display = "none";
}
</script>
<script language="JavaScript1.2">
function rememberMe(cbox)
{
userIDValue = document.forms(0).userID.value;
userPWDValue = document.forms(0).password.value;
userLocale = document.forms(0).userLocale.value;
userBU = document.forms(0).userBU.value;
if (cbox.checked)
{
document.forms(0).rememberFlag.value = "true";
setCookie("userID", userIDValue);
setCookie("userPWD", userPWDValue);
setCookie("userLocale", userLocale);
setCookie("userBU", userBU);
}
}
//function to reset the locale/bu
function setUserLocale(userLocale, userBU)
{
setCookie("userBU", userBU);
setCookie("userLocale", userLocale);
document.forms(0).submit();
}
function submitForm() {
userIDValue = new String();
userIDValue = document.forms(0).userID.value;
userPWDValue = document.forms(0).password.value;
userLocale = document.forms(0).userLocale.value;
userBU = document.forms(0).userBU.value;
rFlag = document.forms(0).rememberFlag.value;
if (document.forms(0).password.value.length <1)
{
//alert("Please enter password");
//alert("Please enter password");
}
else if (rFlag = "true")
{
setCookie("userID", userIDValue);
setCookie("userPWD", userPWDValue);
setCookie("userLocale", userLocale);
setCookie("userBU", userBU);
}
document.forms(0).submit();
}
function _HpbJumpURL(url)
{
if (url != '')
{
window.location = url;
}
}
function checkBrowser(){
var userID=getCookie("userID");
var userPWD = getCookie("userPWD");
var userLocale = getCookie("userLocale");
var userBU = getCookie("userBU");
if (userID != null && userPWD != null)
{
document.forms(0).userID.value = userID;
document.forms(0).password.value = userPWD;
}
this.ver=navigator.appVersion
this.dom=document.getElementById?1:0
//Changes made by ANANDA5 for IE8 Migration
//this.ie6=(this.ver.indexOf("MSIE 6")>-1 && this.dom)?1:0;
this.ie8=(this.ver.indexOf("MSIE 8")>-1 && this.dom)?1:0;
//End
var hName= window.location.host;
var hPlatform = "";
if (hName.indexOf("cert") != -1 )
{
hPlatform = "Certification";
}
else if (hName.indexOf("prod") != -1)
{
hPlatform = "Production";
}
else if (hName.indexOf("int") != -1)
{
hPlatform = "Development";
}
else if (hName.indexOf("lngcentral") != -1)
{
hPlatform = "Production";
}
else
{
hPlatform = "Development";
}
document.login.servName.value=hPlatform;
return this
}
function addToFavorites()
{
window.external.AddFavorite(window.url, "EdSched");
}
function showNew()
{
var userBU = getCookie("userBU");
if (userBU != null && userBU != "LNFR")
{
var newPage=getCookie("newPage3-17");
if (newPage == null || newPage=="TRUE")
{
whatsNew =GB_showFullScreen("What's new for this Release","../whatsnew.jsp");
}
}
}
</script>
</head>
<body onload="checkBrowser(); showNew();">
<form name="login" method="post" action="/ws/ess/default.jsp"><!-- Set the Default Locale -->
<br>
<br>
<br>
<center>
<table style='background: #dae0d2 url("images/bg.gif") repeat-x bottom; border: 1px thin'>
<tbody>
<tr>
<td align="center"><img src="/ws/ess/images/edsched.png" border="0"></td>
</tr>
<tr>
<td align="center"><label class="TextLabel4">(Editorial Scheduling System)</label> <br>
<br>
</td>
</tr>
<tr>
<td align="center"><label class="TextLabel4">Enter the Login Details</label> <br>
</td>
</tr>
</tbody>
</table>
</center>
<center>
<table style='background: #dae0d2 url("images/bg.gif") repeat-x bottom; border: 1px thin'>
<tbody>
<tr>
<td>
<div id="userDetails">
<center>
<table>
<tbody>
<tr class="EbTopBar">
</tr>
<tr class="EbTopBar">
<td align="right" width="100"><label class="TextLabel1">UserId :</label></td>
<td width="110"><input type="text" name="userID" value=""
class="EbInputBg" size="15" tabindex="1"></td>
</tr>
<tr class="EbTopBar">
<td align="right" width="100"><label class="TextLabel1">Password :</label></td>
<td width="110"><input type="password" name="password" value=""
class="EbInputBg" size="15" tabindex="2"></td>
</tr>
</tbody>
</table>
</center>
</div>
</td>
</tr>
<tr>
<td>
<div style="display:" id="vanisher">
<center>
<table>
<tbody>
<tr class="EbTopBar">
<td align="right" width="100"><label class="TextLabel1">Location</label></td>
<td width="110"><select id="userLocale" name="userLocale"
class="ComboBox_ID12">
<OPTION value="US CA" >US CA</OPTION>
<OPTION value="FR" >FR</OPTION>
<OPTION value="AU" >AU</OPTION>
<OPTION value="SNG" >SNG</OPTION>
<OPTION value="LNC" >LNC</OPTION>
</select></td>
</tr>
<tr class="EbTopBar">
<td align="right" width="100"><label class="TextLabel1">Business Unit</label></td>
<td width="110"><select id="userBU" name="userBU"
class="ComboBox_ID12">
<OPTION value="LNC" >LNC</OPTION>
<OPTION value="COS" >COS</OPTION>
<OPTION value="MI" >MI</OPTION>
<OPTION value="MB" >MB</OPTION>
<OPTION value="MDC-SNG" >MDC-SNG</OPTION>
<OPTION value="LNFR" >LNFR</OPTION>
</select></td>
</tr>
<tr class="EbTopBar">
<td/>
<td align="left" width="100"><label class="TextLabel1">Correct business unit must be selected</label></td>
</tr>
</tbody>
</table>
</center>
</div>
</td>
</tr>
<!--
<TR>
<TD CLASS="link"><A HREF="javascript:dispHandle(vanisher)">Choose
Language </A></TD>
</TR>
-->
<tr>
<td>
<center>
<table>
<tbody>
<tr>
<td align="right" width="100"><label class="TextLabel1">Platform :</label></td>
<td width="110"><input type="text" name="servName" value=""
class="EbReadOnly" size="15" tabindex="3" readonly></td>
</tr>
<tr>
<td align="right" width="100"></td>
<!--
<TD WIDTH="110"><INPUT TYPE="checkbox" NAME="rememberme" VALUE="N"
ONCLICK="rememberMe(this);"><LABEL CLASS="TextLabel1">Remember Me</LABEL></TD>
-->
</tr>
</tbody>
</table>
</center>
</td>
</tr>
<tr>
<td>
<center>
<table id="loginButtons">
<tbody>
<tr>
<td width="80"></td>
<td><input type="submit" name="login"
value='Login'
onclick="submitForm()" class="EbButton" /></td>
<td><input type="reset" name="cancel"
value='Cancel' class="EbButton" /></td>
<td width="80"></td>
</tr>
</tbody>
</table>
</center>
</td>
</tr>
</tbody>
</table>
</center>
<input type="hidden" name="rememberFlag" value="false">
<center><a href="/ws/ess/frontpage/bugs.jsp">Login problems?</a><br>
</center>
<center>
<table border="1">
<tbody align="center">
<tr>
<td align="center"><a href="javascript:setUserLocale('US', 'MB');">English</a></td>
<td align="center"><a href="javascript:setUserLocale('FR', 'LNFR')";>Français</a></td>
</tr>
</tbody>
</table>
</center>
<input name="hiddenHost" type="hidden" value='cpc44743'/>
</form>
</body>
</html>