问题

Question

问题

我正在尝试抓取图片并自动分页。我正在使用页面上项目的@IBAction buttonAction(sender: UIButton) { Alamofire.request("MYURL.com").responseJSON { (response) in if let JSON : NSDictionary = response.result.value as! NSDictionary? { let classNameArray = JSON["className"] as! NSArray print("---") print(classNameArray) let theClassChooseViewController = storyboard.instantiateViewController(withIdentifier:"ClassChooseViewController" as ClassChooseViewController theClassChooseViewController.classNameArray = classNameArray presentViewController(theClassChooseViewController, animated: true, completion: nil) } } }描述与总项目span。我想让梦魇经历这个while循环，但是它会挂起并给我一个1 - 20 of 83,829 results错误。有没有办法每次都执行此操作而不是推入堆栈，因为我觉得它是这样做的。

要修复的代码

Javascript heap out of memory

这是与function scrapeEach(paginate) { // while paginate has next scrapeEach while (paginate.next) { nightmare .wait(4000) .realClick('#pageNext .results-next') .wait(4000) .evaluate(() => { return document.body.innerHTML }) .then(result => { scrapeImages(result); paginate.update(); paginate.state(); }) .catch(error => { console.error('Search failed:', error); }); } return nightmare.end() }一起使用的额外代码，我创建了一个分页对象来跟踪这样的页面：

scrapeEach()

这会将图片从一页上删除

function Paginate(url, pgd) {

  this.url = url;
  this.array = pgd.split(" ");

  this.currentPage = Number(this.array[0]);
  this.totalItems = Number(_.join(_.filter(this.array[4], char => char != ","), ''));
  this.itemsPerPage = Number(this.array[2]);
  this.totalPages = Math.floor(this.totalItems / this.itemsPerPage)
  this.next = true;

  this.update = () => {
   this.currentPage += 1;
   if (this.currentPage >= this.totalPages)
     this.next = false;
   }

  this.state = () => {
    console.log("-------- Pagination ----------")
    console.log("current page: " + this.currentPage);
    console.log("total pages: " + this.totalPages);
    console.log("total items: " + this.totalItems);
    console.log("items per page: " + this.itemsPerPage);
    console.log("has next page: " + this.next);
    console.log("------------------------------\n");
  }
}

此功能启动整个过程

// scrapes all image data on one page and updates to db
function scrapeImages(html) {
  xr(html, '#returns > li', [
    {
     img: 'dl.return-art > dd > a > img@src',
     title: 'dl.return-art > dt > a@html',
     created: 'dl.return-art > .created',
     medium: 'dl.return-art > .medium',
     dimensions: 'dl.return-art > .dimensions',
     credit: 'dl.return-art > .credit',
     accession: 'dl.return-art > .accession'
    }
  ])((err, res) => {
   if (err)
     throw err;
   Artwork.addArt(res);
 })
}

错误消息

// the onview endpoint
function onView() {
  nightmare.goto(config.NGA.online)
  nightmare
    .wait(3000)
    .evaluate(() => {
       return [document.location.href, document.querySelector('span.results-span').innerHTML]
    })
    .then(([url, pgd]) => scrapeEach(new Paginate(url, pgd)))
  }

Answer 1

所以我明白了。问题是异步问题，while循环异步运行直到完成，然后恶梦实例永远不会运行。我更新了我的解决方案，从自动点击到下一页，计算网址&pageNumber=中的下一页，并使用此网站的自定义页面对象传递有关页面，网址和页面上的项目的数据。我还添加了一些调试信息来显示。

function Paginate(url, pgd) {

 this.url = url;
 this.array = pgd.split(" ");

 this.currentPage = Number(this.array[0]);
 this.totalItems = Number(_.join(_.filter(this.array[4], char => char != ","), ''));
 this.itemsPerPage = Number(this.array[2]);
 this.totalPages = Math.floor(this.totalItems / this.itemsPerPage)
 this.next = true;

 this.update = () => {
   let chunks = url.split("&").filter(segment => !segment.includes('Number='));

   this.currentPage += 1;
    if (this.currentPage >= this.totalPages)
      this.next = false;

   this.url = _.join(chunks, "") + '&pageNumber=' + this.currentPage;
 }

 this.state = () => {
   console.log("-------- Pagination ----------")
   console.log("current page: " + this.currentPage);
   console.log("total pages: " + this.totalPages);
   console.log("total items: " + this.totalItems);
   console.log("items per page: " + this.itemsPerPage);
   console.log("has next page: " + this.next);
   console.log("current url: " + this.url);
   console.log("------------------------------\n");
 }

}

我使用async的whilst同步使用while循环，并在每次迭代时执行噩梦。

function scrapeEach(paginate) {
  // while paginate has next scrapeEach
  let hasNext = () => paginate.next && paginate.currentPage < 10
  async.whilst(hasNext, next => {

   nightmare
     .goto(paginate.url)
     .wait(4000)
     .evaluate(() => {
        return document.body.innerHTML
     })
     .then(result => {
        scrapeImages(result);
        paginate.update();
        paginate.state();
        next();
     })
     .catch(error => {
        console.error('Search failed:', error);
     });
   }, err => {
    if (err)
      throw err;
    console.log("finished!");
  })
  return nightmare;
 }

Answer 2

限制并行运行的进程

    var limit   = 10;       // concurrent read // this can be increased
    var running = 0; 

    function scrapeEach(paginate) {
     // while paginate has next scrapeEach
     while (paginate.next && running < limit) {
       running++;
       nightmare
         .wait(4000)
         .realClick('#pageNext .results-next')
         .wait(4000)
         .evaluate(() => {
           return document.body.innerHTML
         })
         .then(result => {

            scrapeImages(result , function(){

             paginate.update();
             paginate.state();
             running--;

            });
         })
         .catch(error => {
           console.error('Search failed:', error);
           running--;
         });
      }
    return nightmare.end()
    }


   // scrapes all image data on one page and updates to db
    function scrapeImages(html ,cb) {
      xr(html, '#returns > li', [
        {
         img: 'dl.return-art > dd > a > img@src',
         title: 'dl.return-art > dt > a@html',
         created: 'dl.return-art > .created',
         medium: 'dl.return-art > .medium',
         dimensions: 'dl.return-art > .dimensions',
         credit: 'dl.return-art > .credit',
         accession: 'dl.return-art > .accession'
        }
      ])((err, res) => {
       if (err)
         throw err;
       Artwork.addArt(res);
       cb();
     })
    }

如何在while循环中执行nightmarejs

问题

要修复的代码

错误消息

2 个答案: