实例：node.js 抓取代理ip代码

/*

* 支持：node.js v7.9.0

*/

const cheerio=require('cheerio');

const fetch =require('node-fetch');

const Promise=require('bluebird');

let mongoose=require('mongoose');

Promise.promisifyAll(mongoose);

let Schema=mongoose.Schema;

mongoose.connect('mongodb://localhost:27017/ipproxypool');

let IPpool=newSchema({

  ip:{type:String,unique:true}

})

let Ipproxy=mongoose.model('IP',IPpool);

functionfetchUrl(url){

  fetch(url,{

    method:'get',

    headers:{

    }

  })

  .then(res=>res.text())

  .then(body=>{

    let $=cheerio.load(body);

    let length=$('#list table tbody').find('tr').length;

    for(let i=0;i

    let ipaddress= $('#list table tbody').find('tr').eq(i).find('td').eq(0).text() ;

    let port = $('#list table tbody').find('tr').eq(i).find('td').eq(1).text();

    console.log(`IP:${ipaddress}:${port}`);

    let ip=`${ipaddress}:${port}`

    let ippool=newIpproxy({

      ip:ip

    })

    ippool.save();

    }

  })

}

varsleep =function(time) {

  returnnewPromise(function(resolve, reject) {

    setTimeout(function() {

      resolve('ok');

    }, time);

  })

};

const pageNumber=10;

varstart = asyncfunction(){

  for(let j=1;j

     console.log(`当前是第${j}次等待..`);

    fetchUrl(`http://www.kuaidaili.com/free/inha/${j}/`);

    await sleep(1500);

  }

}

{

 "name":"demo-4-ipproxypool",

 "version":"1.0.0",

 "description":"",

 "main":"index.js",

 "scripts": {

  "test":"echo \"Error: no test specified\" && exit 1"

 },

 "author":"false-l",

 "license":"",

 "devDependencies": {

  "babel-preset-es2015":"^6.24.1",

  "babel-preset-react":"^6.24.1",

  "babel-preset-stage-3":"^6.24.1"

 },

 "dependencies": {

  "babel-core":"^6.24.1",

  "bluebird":"^3.5.0",

  "cheerio":"^0.22.0",

  "koa":"^2.2.0",

  "koa-router":"^7.1.1",

  "mongoose":"^4.9.6",

  "node-fetch":"^1.6.3"

 }

}

const Promise=require('bluebird');

let mongoose=require('mongoose');

const koa=require('koa');

const app=newkoa();

varrouter = require('koa-router')();

Promise.promisifyAll(mongoose);

let Schema=mongoose.Schema;

mongoose.connect('mongodb://localhost:27017/ipproxypool');

let IPpool=newSchema({

  ip:{type:String,unique:true}

})

let Ipproxy=mongoose.model('IP',IPpool);

app.use(async (ctx, next) => {

 await next();

 vardata=await Ipproxy.find({},function(err,ips){

  varipmap=[];

   ips.forEach(function(ip){

     ipmap[ip._id]=ip;

     //console.log(ip)

   });

 })

 varmap=data.map(ip=>ip.ip);

 ctx.response.type ='text/json'

 ctx.response.body = map;

});

app.listen(3000);

console.log('server listen:3000')