Web爬虫Apache Nutch内置了对NTLM的支持。我正在尝试使用1.7版本来使用NTLM身份验证来抓取网站（Windows Sharepoint）。我根据https://wiki.apache.org/nutch/HttpAuthenticationSchemes设置了Nutch，这尤其意味着我有凭据
<credentials username="rickert" password="mypassword">
<authscope host="server-to-be-crawled.com" port="80" realm="CORP" scheme="NTLM"/>
</credentials>
配置。当我查看日志文件时，我可以看到Nutch尝试访问种子URL并经历“正常”NTLM循环：在第一次GET期间获取401错误，解压缩NTLM挑战并在下一次GET中发送NTLM身份验证（使用保持连接）。但是，第二次GET也没有成功。
当我怀疑我的凭据或特定设置存在一些基本问题时，我就是这样：我在Windows主机上的Debian guest虚拟机盒中运行Nutch。但令我惊讶的是wget和curl能够使用我的凭据从Debian来宾中检索文档。有趣的是，两个命令行工具只需要用户名和密码即可工作。另一方面，完整的承诺NTLM规范还需要主机和域。根据规范， host 是请求源自我将解释为运行http代理的那个，Windows域中的域用户名与之关联。我的假设是两个工具都只是将这些细节留空。
这是Nutch配置的来源： host 据称在配置文件中以http.agent.host的形式提供。 域应该被配置为凭据的域，但文档却说这是一个约定而不是真正必要的。但是，无论我是否设置了一个领域，结果都是一样的。再次查看日志文件，无论我使用哪个领域，我都可以看到使用<any_realm>@server-to-be-crawled.com解析身份验证的一些消息。
我的直觉是Nutch配置值有一些错误的映射到执行GET的Java类httpclient所需的NTLM参数。我很无奈。任何人都可以给我一些提示，如何进一步调试这个？有没有人有一个适用于SharePoint Server的具体配置？谢谢！
答案 0 :(得分：0)
这是一个旧线程，但这似乎是一个常见问题，我终于找到了解决方法。
在我的情况下，问题是我尝试爬网的内容源托管在相当最新的IIS服务器上。检查标题表明它正在使用NTLMv1，但是在阅读到Apache Commons HttpClient v3.x仅支持NTLMv1而不支持NTLMv2之后，我去寻找一种方法来将支持添加到v1.15，而不升级到较新的HttpComponents版本。 HttpClient。
线索在documentation for the newer HC version of HttpClient中 因此，使用this approach with JCIFS，我设法修改了nutst协议-httpclient Http类，以便它使用了基于JCIFS的新NTLM方案进行身份验证。步骤如下：
完成工作后，我便能够对受NTLMv2保护的网站进行爬网。
还添加了许多额外的日志记录，然后我可以看到身份验证握手详细信息，该信息表明实际上正在使用NTLMv2。
Http.configureClient中的更改如下：
/** Configures the HTTP client */
private void configureClient() {
LOG.info("Setting new NTLM scheme: " + JcifsNtlmScheme.class.getName());
AuthPolicy.registerAuthScheme(AuthPolicy.NTLM, JcifsNtlmScheme.class);
...
}
新的NTLM方案实现看起来像这样（需要整理一下）。
public class JcifsNtlmScheme implements AuthScheme {
public static final Logger LOG = LoggerFactory.getLogger(JcifsNtlmScheme.class);
/** NTLM challenge string. */
private String ntlmchallenge = null;
private static final int UNINITIATED = 0;
private static final int INITIATED = 1;
private static final int TYPE1_MSG_GENERATED = 2;
private static final int TYPE2_MSG_RECEIVED = 3;
private static final int TYPE3_MSG_GENERATED = 4;
private static final int FAILED = Integer.MAX_VALUE;
/** Authentication process state */
private int state;
public JcifsNtlmScheme() throws AuthenticationException {
// Check if JCIFS is present. If not present, do not proceed.
try {
Class.forName("jcifs.ntlmssp.NtlmMessage", false, this.getClass().getClassLoader());
LOG.trace("jcifs.ntlmssp.NtlmMessage is present");
} catch (ClassNotFoundException e) {
throw new AuthenticationException("Unable to proceed as JCIFS library is not found.");
}
}
public String authenticate(Credentials credentials, HttpMethod method) throws AuthenticationException {
LOG.trace("authenticate called. State: " + this.state);
if (this.state == UNINITIATED) {
throw new IllegalStateException("NTLM authentication process has not been initiated");
}
NTCredentials ntcredentials = null;
try {
ntcredentials = (NTCredentials) credentials;
} catch (ClassCastException e) {
throw new InvalidCredentialsException(
"Credentials cannot be used for NTLM authentication: " + credentials.getClass().getName());
}
NTLM ntlm = new NTLM();
String charset = method.getParams().getCredentialCharset();
LOG.trace("Setting credential charset to: " + charset);
ntlm.setCredentialCharset(charset);
String response = null;
if (this.state == INITIATED || this.state == FAILED) {
LOG.trace("Generating TYPE1 message");
response = ntlm.generateType1Msg(ntcredentials.getHost(), ntcredentials.getDomain());
this.state = TYPE1_MSG_GENERATED;
} else {
LOG.trace("Generating TYPE3 message");
response = ntlm.generateType3Msg(ntcredentials.getUserName(), ntcredentials.getPassword(),
ntcredentials.getHost(), ntcredentials.getDomain(), this.ntlmchallenge);
this.state = TYPE3_MSG_GENERATED;
}
String result = "NTLM " + response;
return result;
}
public String authenticate(Credentials credentials, String method, String uri) throws AuthenticationException {
throw new RuntimeException("Not implemented as it is deprecated anyway in Httpclient 3.x");
}
public String getID() {
throw new RuntimeException("Not implemented as it is deprecated anyway in Httpclient 3.x");
}
/**
* Returns the authentication parameter with the given name, if available.
*
*
* There are no valid parameters for NTLM authentication so this method always
* returns null.
*
*
* @param name The name of the parameter to be returned
*
* @return the parameter with the given name
*/
public String getParameter(String name) {
if (name == null) {
throw new IllegalArgumentException("Parameter name may not be null");
}
return null;
}
/**
* The concept of an authentication realm is not supported by the NTLM
* authentication scheme. Always returns null.
*
* @return null
*/
public String getRealm() {
return null;
}
/**
* Returns textual designation of the NTLM authentication scheme.
*
* @return ntlm
*/
public String getSchemeName() {
return "ntlm";
}
/**
* Tests if the NTLM authentication process has been completed.
*
* @return true if Basic authorization has been processed,
* false otherwise.
*
* @since 3.0
*/
public boolean isComplete() {
boolean result = this.state == TYPE3_MSG_GENERATED || this.state == FAILED;
LOG.trace("isComplete? " + result);
return result;
}
/**
* Returns true. NTLM authentication scheme is connection based.
*
* @return true.
*
* @since 3.0
*/
public boolean isConnectionBased() {
return true;
}
/**
* Processes the NTLM challenge.
*
* @param challenge the challenge string
*
* @throws MalformedChallengeException is thrown if the authentication challenge
* is malformed
*
* @since 3.0
*/
public void processChallenge(final String challenge) throws MalformedChallengeException {
String s = AuthChallengeParser.extractScheme(challenge);
LOG.trace("processChallenge called. challenge: " + challenge + " scheme: " + s);
if (!s.equalsIgnoreCase(getSchemeName())) {
LOG.trace("Invalid scheme name in challenge. Should be: " + getSchemeName());
throw new MalformedChallengeException("Invalid NTLM challenge: " + challenge);
}
int i = challenge.indexOf(' ');
if (i != -1) {
LOG.trace("processChallenge: TYPE2 message received");
s = challenge.substring(i, challenge.length());
this.ntlmchallenge = s.trim();
this.state = TYPE2_MSG_RECEIVED;
} else {
this.ntlmchallenge = "";
if (this.state == UNINITIATED) {
this.state = INITIATED;
LOG.trace("State was UNINITIATED, switching to INITIATED");
} else {
LOG.trace("State is FAILED");
this.state = FAILED;
}
}
}
private class NTLM {
/** Character encoding */
public static final String DEFAULT_CHARSET = "ASCII";
/**
* The character was used by 3.x's NTLM to encode the username and password.
* Apparently, this is not needed in when passing username, password from
* NTCredentials to the JCIFS library
*/
private String credentialCharset = DEFAULT_CHARSET;
void setCredentialCharset(String credentialCharset) {
this.credentialCharset = credentialCharset;
}
private String generateType1Msg(String host, String domain) {
jcifs.ntlmssp.Type1Message t1m = new jcifs.ntlmssp.Type1Message(
jcifs.ntlmssp.Type1Message.getDefaultFlags(), domain, host);
String result = jcifs.util.Base64.encode(t1m.toByteArray());
LOG.trace("generateType1Msg: " + result);
return result;
}
private String generateType3Msg(String username, String password, String host, String domain,
String challenge) {
jcifs.ntlmssp.Type2Message t2m;
try {
t2m = new jcifs.ntlmssp.Type2Message(jcifs.util.Base64.decode(challenge));
} catch (IOException e) {
throw new RuntimeException("Invalid Type2 message", e);
}
jcifs.ntlmssp.Type3Message t3m = new jcifs.ntlmssp.Type3Message(t2m, password, domain, username, host, 0);
String result = jcifs.util.Base64.encode(t3m.toByteArray());
LOG.trace("generateType3Msg username: [" + username + "] host: [" + host + "] domain: [" + domain
+ "] response: [" + result + "]");
return result;
}
}
}