CasperJS 循环或迭代多个网页? [英] CasperJS loop or iterate through multiple web pages?

问题描述

我有一个 CasperJS 脚本，可以从一个网页中抓取评级和日期.现在我想从同一个网站下的多个页面中抓取相同的数据.给定此代码，我如何遍历不同的子页面:

I have a CasperJS script that scrapes ratings and dates from one webpage. Now I want to scrape the same data from multiple pages under the same website. How can I loop through the different subpages given this code:

var ratings = [];
var dates = [];
var casper = require('casper').create({

    pageSettings: {
        loadImages:  false,         
        loadPlugins: false          
    },
    logLevel: "debug",             
    verbose: true                   
});

var fs = require('fs');

function getRatings() {
    var ratings = document.querySelectorAll('#BVRRRatingOverall_Review_Display > div.BVRRRatingNormalImage > img');
    return Array.prototype.map.call(ratings, function(e) {
        return e.getAttribute('title');
    });
}

function getDate() {
    var dates = document.querySelectorAll('#BVSubmissionPopupContainer > div.BVRRReviewDisplayStyle5Header > div.BVRRReviewDateContainer > span.BVRRValue.BVRRReviewDate');

    return Array.prototype.map.call(dates, function(e) {

        return e.innerHTML;

    });
}

casper.start('http://www.t-mobile.com/cell-phones/samsung-galaxy-s-5.html?bvrrp=9060/reviews/product/1/598aea53-16d0-4c12-b53a-105157092c52.htm', function(){

    this.echo('hi');
});

casper.then(function() {

    ratings = this.evaluate(getRatings);
    dates = this.evaluate(getDate);

    this.echo(ratings);
});


casper.run(function() {

    this.echo(ratings.length + ' ratings found:');

     for(var i=0; i<ratings.length; i++){
        ratings[i] = ratings[i]+': '+dates[i];
        dates[i] = '';
     }
    this.echo(ratings);
    var content = ratings;

    content = content.join("
");

    fs.write("C:/Users/Karan/Copy/tweesis/implementation/scraping/samsungratings.txt", content, 'w'); 

    this.echo(dates.length + ' dates found:').exit();



});

感谢任何帮助:)

推荐答案

因为有下一页按钮，所以可以递归遍历所有页面:

Since there exists a next page button, you can use it to traverse all pages recursively:

function getRatingsAndWrite(){
    ratings = casper.evaluate(getRatings);
    dates = casper.evaluate(getDate);

    casper.echo(ratings);
    casper.echo(ratings.length + ' ratings found:');

    for(var i=0; i<ratings.length; i++){
        ratings[i] = ratings[i]+': '+dates[i];
        dates[i] = '';
    }
    casper.echo(ratings);
    var content = ratings;

    content = content.join("
");

    fs.write("C:/Users/Karan/Copy/tweesis/implementation/scraping/samsungratings.txt", content, 'a'); 

    casper.echo(dates.length + ' dates found:');

    var nextLink = ".BVRRPageLink.BVRRNextPage > a";
    if (casper.visible(nextLink)) {
        casper.thenClick(nextLink);
        casper.then(getRatingsAndWrite);
    } else {
        casper.echo("END")
    }
}

casper.start('http://www.t-mobile.com/cell-phones/samsung-galaxy-s-5.html?bvrrp=9060/reviews/product/1/598aea53-16d0-4c12-b53a-105157092c52.htm');

casper.then(getRatingsAndWrite);

casper.run();

一个相关的答案是A:CasperJS在按钮点击后解析下一页.

这篇关于CasperJS 循环或迭代多个网页?的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！