javascript - Node - wait for map to finish before continuing - Stack Overflow

I have this file in my node app that supposed to go fetch me some data about every league champion from their official website using cheerio and its going all great but when I add all the data to my array to then return it as json data the write function runs before the map finishes so I just creating a json file with an empty array in it:

const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');

const champions = fs.readFileSync('champions.json');
const championsObj = JSON.parse(champions);

let champsList = [];

championsObj.map(champ => {
  request(champ.href, (err, res, html) => {
    if (!err && res.statusCode == 200) {
      const $ = cheerio.load(html);

      const champName = $('.style__Title-sc-14gxj1e-3 span').text();

      let skins = [];

      const skinsList = $('.style__CarouselItemText-sc-1tlyqoa-16').each(
        (i, el) => {
          const skinName = $(el).text();
          skins.push = skinName;
        }
      );

      const champion = {
        champName,
        skins
      };

      console.log(champion);

      champsList.push = champion;
    }
  });
});

const jsonContent = JSON.stringify(champsList);

fs.writeFile('champions2.json', jsonContent, 'utf8', function(err) {
  if (err) {
    console.log(err);
  }
});

I'm not a node expert but I tried using Promise but it didn't work but I'm not sure maybe I used it wrong.

UPDATE #1: using axios

championsObj.map(async champ => {
  const html = await axios.get(champ.href);
  const $ = await cheerio.load(html);

  const champName = $('.style__Title-sc-14gxj1e-3 span').text();

  let skins = [];

  const skinsList = $('.style__CarouselItemText-sc-1tlyqoa-16').each(
    (i, el) => {
      const skinName = $(el).text();
      skins.push = skinName;
    }
  );

  const champion = {
    champName,
    skins
  };

  console.log(champion);

  champsList.push = champion;
});

I have this file in my node app that supposed to go fetch me some data about every league champion from their official website using cheerio and its going all great but when I add all the data to my array to then return it as json data the write function runs before the map finishes so I just creating a json file with an empty array in it:

const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');

const champions = fs.readFileSync('champions.json');
const championsObj = JSON.parse(champions);

let champsList = [];

championsObj.map(champ => {
  request(champ.href, (err, res, html) => {
    if (!err && res.statusCode == 200) {
      const $ = cheerio.load(html);

      const champName = $('.style__Title-sc-14gxj1e-3 span').text();

      let skins = [];

      const skinsList = $('.style__CarouselItemText-sc-1tlyqoa-16').each(
        (i, el) => {
          const skinName = $(el).text();
          skins.push = skinName;
        }
      );

      const champion = {
        champName,
        skins
      };

      console.log(champion);

      champsList.push = champion;
    }
  });
});

const jsonContent = JSON.stringify(champsList);

fs.writeFile('champions2.json', jsonContent, 'utf8', function(err) {
  if (err) {
    console.log(err);
  }
});

I'm not a node expert but I tried using Promise but it didn't work but I'm not sure maybe I used it wrong.

UPDATE #1: using axios

championsObj.map(async champ => {
  const html = await axios.get(champ.href);
  const $ = await cheerio.load(html);

  const champName = $('.style__Title-sc-14gxj1e-3 span').text();

  let skins = [];

  const skinsList = $('.style__CarouselItemText-sc-1tlyqoa-16').each(
    (i, el) => {
      const skinName = $(el).text();
      skins.push = skinName;
    }
  );

  const champion = {
    champName,
    skins
  };

  console.log(champion);

  champsList.push = champion;
});

• javascript
• node.js
• promise
• es6-promise

Share Improve this question Follow edited Mar 29, 2020 at 0:37 A. Atiyah asked Mar 29, 2020 at 0:14 A. AtiyahA. Atiyah 55511 gold badge66 silver badges1717 bronze badges 5

• the issue with this is that request is asynchronous - and the way you've written it, it's not guaranteed the .map will populate champsList in the right order – Jaromanda X Commented Mar 29, 2020 at 0:17
• so you think axios or fetch might be a better option? – A. Atiyah Commented Mar 29, 2020 at 0:19
• You say you tried promises, but haven't shown that code (fetch and axios use promises) - there's also "request-promise" npm module – Jaromanda X Commented Mar 29, 2020 at 0:26
• one sec i'll update the question with the axios code – A. Atiyah Commented Mar 29, 2020 at 0:36
• 1 read about Promise.all on mdn. This is essential to what you are doing and to node programming in general. You don't need any fancy libraries to do this – Max Commented Mar 29, 2020 at 0:45

Add a ment  | 

2 Answers 2

Sorted by: Reset to default 5

you can use await Promise.all(<array>.map(async () => {...}). it does not require any additional dependencies. however you have no guarantees about the order of asynchronous iterations (starting all the iterations in the right order, but no guarantees about iterations' endings).

Your problem here is that Array#map doesn't wait for asynchronous functions such as the request calls to finish before moving on. I remend p-map with got. To ensure perfect execution order, I also remend reading and writing the file asynchronously.

const got = require('got');
const pMap = require('p-map');
const cheerio = require('cheerio');
const fs = require('fs').promises;

(async () => {
    const champions = JSON.parse(await fs.readFile('champions.json', 'utf8'));

    let champsList = await pMap(champions, async champ => {
        const {
            body
        } = await got(champ.href)

        const $ = cheerio.load(body);

        const champName = $('.style__Title-sc-14gxj1e-3 span').text();

        let skins = [];

        $('.style__CarouselItemText-sc-1tlyqoa-16').each(
            (_, el) => {
                const skinName = $(el).text();
                skins.push(skinName);
            }
        );

        const champion = {
            champName,
            skins
        };

        console.log(champion);

        return champion;
    })

    await fs.writeFile('champions2.json', JSON.stringify(champsList));
})();