index.js

let request = require('request');
let cheerio = require('cheerio');
let async = require ('async');
let URL = require('url-parse');
let fs = require('fs');
let logger = fs.createWriteStream('log.txt', {
  flags: 'a'
})

var base = "http://www.lespagesjaunesafrique.com"
var origin = "http://www.lespagesjaunesafrique.com"
var urls = [];
var database = [];
var final = [];

const multiRequest = 10

console.log("Visiting page " + base);
crawl(base, '/pays')

function collectInternalLinks($, pattern) {
  let allRelativeLinks = [];

  let relativeLinks = $("a[href^='/']");
  relativeLinks.each(function() {
  	if ($(this).attr('href').indexOf(pattern) === 0)
	    allRelativeLinks.push($(this).attr('href'));
  })
  return allRelativeLinks;
}

function collectInternalLinks2($, pattern) {
  let allRelativeLinks = [];

  let relativeLinks = $("a[href^='/']");
  relativeLinks.each(function() {
  	if ($(this).attr('href').indexOf(pattern) === 0) {
  		let toPush = {}
  		toPush.category = $(this).parent().prev().children().children().text().trim()
  		toPush.url = $(this).attr('href')
  		urls.push(toPush)
  	}
  })

  return allRelativeLinks;
}

function crawl(url, a_pattern) {
	let country = []
	request(url, function(error, response, body) {
		if (error)
			console.log("Error: " + error);
		if (response.statusCode != 200) {
			console.log(response.statusCode)
			return false
		}
		let $ = cheerio.load(body);
		country = collectInternalLinks($, a_pattern)
		console.log('Etape 0 finished');
		async.eachOfLimit(country, multiRequest, crawlCountry, function(err) {
			if (err)
	      console.log(err);
	    else {
	      console.log('Etape 1 finished');
	      async.eachOfLimit(urls, multiRequest, visit_urls, function(err) {
					if (err)
			      console.log(err);
			    else {
			      console.log('Etape 2 finished');
			      console.log(database)
			      async.eachOfLimit(database, multiRequest, retrieve_email, function(err) {
							if (err)
					      console.log(err);
					    else {
					      console.log('Etape 3 finished');
					      logger.write(JSON.stringify(final))
					    }
						})
			    }
				})
	    }
		})
	})
}

function crawlCountry(url, i, callback) {
	request(origin + url, function(err, response, body) {
		if (err)
			callback(origin + url + '\n' + err)
		else if (response.statusCode != 200) {
			console.log('error', response.statusCode)
			callback(null)
		}
		else {
			let $ = cheerio.load(body);
			collectInternalLinks2($, "/societes", true)
			callback(null)
		}
	})
}

function searchForPagination($) {
	let allRelativeLinks = [];

  let relativeLinks = $("a[href^='/']");
  relativeLinks.each(function() {
  	if ($(this).attr('href').indexOf('/start-') > 0) {
  		if (urls.indexOf($(this).attr('href')) < 0)
				urls.push($(this).attr('href'));
  	}
  });

  return allRelativeLinks;
}
function visit_urls(url, i, callback){
	request(origin + url.url, function(err, response, body) {
		if (err) {
			console.log(origin + url.url + '\n' + err)
			callback(null)
			// callback(origin + url.url + '\n' + err)
		}
		else if (response.statusCode != 200) {
			console.log('error', response.statusCode)
			callback(null)
		}
		else {
			let $ = cheerio.load(body);
			searchForPagination($)
			console.log('Etape 2', origin + url.url, Math.round((i / urls.length) * 100) + '%')
			$('.ct-main-text').each(function(i) {
				let emailLink = $(this).children('.ct-product--description').children('.buttonShowCo').attr().onclick
				emailLink = emailLink.substring(21)
				emailLink = emailLink.substring(0, emailLink.indexOf("'"));
				let ar = {
					"country": $('h3').text().trim(),
					"category": url.category,
					"sub_category": $('h4').text().trim(),
					"name": $(this).children('.ct-product--tilte').text().trim(),
					"website": $(this).children('.ct-product--description').children('a').text().trim(),
					"emailLink": emailLink,
					"phone": "",
					"email": ""
				}
				database.push(ar)
			})
			callback(null)
		}
	})
	return false
}

function retrieve_email(database_line, i, callback) {
	request(origin + encodeURL(database_line.emailLink), function(err, response, body) {
		if (err) {
			console.log(origin + encodeURL(database_line.emailLink) + '\n' + err)
			callback(null)
			// callback(origin + encodeURL(database_line.emailLink) + '\n' + err)
		}
		else if (response.statusCode != 200) {
			console.log(response.statusCode, origin + encodeURL(database_line.emailLink))
			callback(null)
		}
		else {
			let $ = cheerio.load(body);
			console.log('Etape 3', Math.round((i / database.length) * 100) + '%')
			let email = null;
			if ($('a').text().trim()) {
				email = $('a').text().trim()
				let nbr = $(body).text().trim()
				nbr = nbr.replace(email, '').trim()
				database_line['email'] = email;
				database_line['phone'] = nbr
				delete database_line.emailLink
				final.push(database_line)
			}
			callback(null)
		}
	})
	return false
}

function encodeURL(s) {
  return encodeURI(s).replace(/['’]/g, function(c) {
    return '%' + c.charCodeAt(0).toString(16);
  });
}