node.js Get.Request & Pagination & Async

问题

I'm having a tremendously tough time organizing the flow here as I'm self-taught so wondering if someone might be able to assist.

var channelIds = ['XYZ','ABC','QRS']
var playlistIds = [];
var videoIds = [];

ORDER OF PROCESS

1. Get All Playlist IDs: If returning Get Request JSON contains nextPageToken run Get Request again with that page before going to (2)

2. Get All Video IDs: If returning Get Request JSON contains nextPageToken run Get Request again with that page before going to (3)

3. Aggregate into Final Array: I need put all in an array such as: var ArrFinal = [{channelId,playlistID,videoId},{channelId,playlistID,videoId},{channelId,playlistID,videoId}];

I don't necessarily need someone to write the whole thing. I'm trying to better understand the most efficient way to know when the previous step is done, but also handle the nextPageToken iteration.

回答1:

i'm not familiar with the youtube api. But what you basically need is a get function for each endpoint. This function should also care about the "nextPageToken".

Something like that: (not tested)

'use strict';

const Promise = require('bluebird');
const request = Promise.promisifyAll(require('request'));

const playlistEndpoint = '/youtube/v3/playlists';
const baseUrl = 'https://www.googleapis.com'

const channelIds = ['xy', 'ab', 'cd'];

const getPlaylist = async (channelId, pageToken, playlists) => {

  const url = `${baseUrl}${playlistEndpoint}`;
  const qs = { 
    channelId,
    maxResults: 25,
    pageToken
  };

  try {
    const playlistRequest = await request.getAsync({ url, qs });
    const nextPageToken = playlistRequest.body.nextPageToken;
    // if we already had items, combine with the new ones
    const items = playlists ? playlists.concat(playlistRequest.body.items) : playlistRequest.body.items;
    if (nextPageToken) {
      // if token, do the same again and pass results to function
      return getPlaylist(channelId, nextPageToken, items);
    }
    // if no token we are finished 
    return items;
  }
  catch (e) {
    console.log(e.message);
  }
};

const getVideos = async (playlistId, pageToken, videos) => {

  // pretty much the same as above
}

function awesome(channelIds) {

  const fancyArray = [];

  await Promise.map(channelIds, async (channelId) => {

    const playlists = await getPlaylist(channelId);

    const videos = await Promise.map(playlists, async (playlistId) => {

      const videos = await getVideos(playlistId);

      videos.forEach(videoId => {
        fancyArray.push({ channelId, playlistId, videoId })
      })
    });
  });

  return fancyArray;
}

awesome(channelIds)

// UPDATE

This may be a lot concurrent requests, you can limit them by using

Promise.map(items, item => { somefunction() }, { concurrency: 5 });

来源：https://stackoverflow.com/questions/49117253/node-js-get-request-pagination-async

标签

node.js

youtube-data-api