我的问题是关于从特定网站抓取数据的可能性。 目前，我的算法是将HTML转换为文本，然后检查文件中包含的标记词， 并总结标志的数量。
我的问题在于，在抓取网站时无法“向下滚动”。 如您所见，它正在检查Twitter帐户上的标志数量，但仅限于50sh最新的推文。 我希望我能说清楚。
ps：我以twitter为例，我不是在寻找特定于twitter的东西，而是更强大的东西。
任何提示，我将不胜感激。
样本输出：
DHS和其他代理商：0个实例
国内安全：1个实例
危险品和核武器：0个实例
关注健康+ H1N1：0个实例
基础结构安全性：2个实例
西南边境暴力：1起
恐怖主义：0起
天气/灾难/紧急情况：2次
网络安全：0个实例
标志总数：6个实例
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.*;
import java.util.LinkedList;
import java.util.List;
public class Main {
static List<String> dhsAndOtherAgencies = new LinkedList<>();
static List<String> domesticSecurity = new LinkedList<>();
static List<String> hazmatNuclear = new LinkedList<>();
static List<String> healthConcern = new LinkedList<>();
static List<String> infrastructureSecurity = new LinkedList<>();
static List<String> southwestBorderViolence = new LinkedList<>();
static List<String> terrorism = new LinkedList<>();
static List<String> weatherDisasterEmergency = new LinkedList<>();
static List<String> cyberSecutiry = new LinkedList<>();
static String stream;
public static void main(String[] args) throws IOException {
createLists();
createStream();
Raport raport = generateReport();
System.out.println(raport);
}
static int flagStream(List<String> list, String stream){
int counter = 0;
for (String flag: list){
if(stream.contains(flag)) {
System.out.println(flag);
counter++;
}
}
return counter;
}
static Raport generateReport(){
return new Raport(
flagStream(dhsAndOtherAgencies,stream),
flagStream(domesticSecurity,stream),
flagStream(hazmatNuclear,stream),
flagStream(healthConcern,stream),
flagStream(infrastructureSecurity,stream),
flagStream(southwestBorderViolence,stream),
flagStream(terrorism,stream),
flagStream(weatherDisasterEmergency,stream),
flagStream(cyberSecutiry,stream)
);
}
static void createStream() throws IOException {
Document doc = Jsoup.connect("https://twitter.com/realDonaldTrump").userAgent("mozilla/17.0").get();
stream = doc.text();
}
static void createLists() throws IOException {
BufferedReader read = new BufferedReader(new FileReader("clearListAllCases.txt"));
String input;
int hashCounter = 0;
while((input=read.readLine())!=null){
if(input.charAt(0)=='#'){
hashCounter++;
continue;
}
switch (hashCounter){
case 1:
dhsAndOtherAgencies.add(input);
break;
case 2:
domesticSecurity.add(input);
break;
case 3:
hazmatNuclear.add(input);
break;
case 4:
healthConcern.add(input);
break;
case 5:
infrastructureSecurity.add(input);
break;
case 6:
southwestBorderViolence.add(input);
break;
case 7:
terrorism.add(input);
break;
case 8:
weatherDisasterEmergency.add(input);
break;
case 9:
cyberSecutiry.add(input);
break;
}
}
}
}
class Raport {
int a,b,c,d,e,f,g,h,i;
int totalFlags;
Raport(int a, int b, int c, int d, int e, int f, int g, int h, int i){
this.a = a;
this.b = b;
this.c = c;
this.d = d;
this.e = e;
this.f = f;
this.g = g;
this.h = h;
this.i = i;
totalFlags = a+b+c+d+e+f+g+h+i;
}
public String toString(){
return "DHS & Other Agencies:\t\t\t"+a+" instances\n"+
"Domestic security:\t\t\t\t"+b+" instances\n"+
"HAZMAT & Nuclear:\t\t\t\t"+c+" instances\n"+
"Health Concern + H1N1:\t\t\t"+d+" instances\n"+
"Infrastructure Security:\t\t"+e+" instances\n"+
"Southwest Border Violence:\t\t"+f+" instances\n"+
"Terrorism:\t\t\t\t\t\t"+g+" instances\n"+
"Weather/Disaster/Emergency:\t\t"+h+" instances\n"+
"Cyber Security:\t\t\t\t\t"+i+" instances\n"+
"TOTAL FLAGS:\t\t\t\t\t"+totalFlags+" instances";
}
}