我正在尝试使用rvest执行网络抓取,以从网上商店抓取特定产品类别上的数据。产品结果显示在多个网页上。当我使用我的代码时,我只会得到与第一页上的产品相同的前24个结果。如何调整代码以在所有结果页面上抓取?
谢谢!
url_bol <- 'https://www.bol.com/nl/l/lichtbronnen/N/14483'
webpage_bol <- read_html(url_bol,na.strings=" ",header=TRUE)
head(webpage_bol)
product_title_data_html <- html_nodes(webpage_bol, '.product-title')
product_title_data <- html_text(product_title_data_html)
head(product_title_data)
product_title_data<-gsub("\n","",product_title_data)
product_title_data<-gsub(" ","",product_title_data)
head(product_title_data)
length(product_title_data)
product_brand_data_html <- html_nodes(webpage_bol, '.product-creator')
product_brand_data <-html_text(product_brand_data_html)
head(product_brand_data)
product_brand_data<-gsub("\n","",product_brand_data)
product_price_data<-gsub(" ","",product_price_data)
head(product_brand_data)
length(product_brand_data)
product_price_data_html <- html_nodes(webpage_bol, '.promo-price')
product_price_data <- html_text(product_price_data_html)
head(product_price_data)
product_price_data<-gsub("\n","",product_price_data)
product_price_data<-gsub(" ","",product_price_data)
head(product_price_data)
product_price_data
length(product_price_data)
bol.df <- data.frame(Procuct_title = product_title_data, Brand = product_brand_data, Price = product_price_data)
View(bol.df)
答案 0 :(得分:0)
url<-"https://www.bol.com/nl/l/lichtbronnen/N/14483/?page=3&view=list"
library(rvest)
page<-html_session(url)
total<-html_nodes(page,xpath='//*[@id="js_list_view"]/div[2]/p') %>% html_text()
total<-as.numeric(gsub("[^\\d]+", "", total, perl=TRUE))
all_df<-0
library(data.table)
for(i in 1:ceiling(total/24)){
url_bol <- paste0('https://www.bol.com/nl/l/lichtbronnen/N/14483/?view=list&page=',i)
webpage_bol <- read_html(url_bol,na.strings=" ",header=TRUE)
head(webpage_bol)
product_title_data_html <- html_nodes(webpage_bol, '.product-title')
product_title_data <- html_text(product_title_data_html)
head(product_title_data)
product_title_data<-gsub("\n","",product_title_data)
product_title_data<-gsub(" ","",product_title_data)
head(product_title_data)
length(product_title_data)
product_brand_data_html <- html_nodes(webpage_bol, '.product-creator')
product_brand_data <-html_text(product_brand_data_html)
head(product_brand_data)
product_brand_data<-gsub("\n","",product_brand_data)
product_price_data<-gsub(" ","",product_price_data)
head(product_brand_data)
length(product_brand_data)
product_price_data_html <- html_nodes(webpage_bol, '.promo-price')
product_price_data <- html_text(product_price_data_html)
head(product_price_data)
product_price_data<-gsub("\n","",product_price_data)
product_price_data<-gsub(" ","",product_price_data)
head(product_price_data)
product_price_data
length(product_price_data)
bol.df <- data.frame(Procuct_title = product_title_data, Brand = product_brand_data, Price = product_price_data)
all_df[i]<-list(bol.df)
}
final<-rbindlist(all_df,fill = TRUE)
View(final)