(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.ytSearch = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i MAX_RETRY_ATTEMPTS) { return callback(err, data); } else { // retry debug(' === '); debug(' RETRYING: ' + _options._attempts); debug(' === '); var n = _options._attempts; var wait_ms = Math.pow(2, n - 1) * RETRY_INTERVAL; setTimeout(function () { search(retryOptions, callback); }, wait_ms); } } else { return callback(err, data); } } // override userAgent if set ( not recommended ) if (_options.userAgent) _userAgent = _options.userAgent; // support common alternatives ( mutates ) _options.search = _options.query || _options.search; // initial search text ( _options.search is mutated ) _options.original_search = _options.search; // ignore query, only get metadata from specific video id if (_options.videoId) { return getVideoMetaData(_options, callback_with_retry); } // ignore query, only get metadata from specific playlist id if (_options.listId) { return getPlaylistMetaData(_options, callback_with_retry); } if (!_options.search) { return callback(Error('yt-search: no query given')); } work(); function work() { getSearchResults(_options, callback_with_retry); } } function _videoFilter(video, index, videos) { if (video.type !== 'video') return false; // filter duplicates var videoId = video.videoId; var firstIndex = videos.findIndex(function (el) { return videoId === el.videoId; }); return firstIndex === index; } function _playlistFilter(result, index, results) { if (result.type !== 'list') return false; // filter duplicates var id = result.listId; var firstIndex = results.findIndex(function (el) { return id === el.listId; }); return firstIndex === index; } function _channelFilter(result, index, results) { if (result.type !== 'channel') return false; // filter duplicates var url = result.url; var firstIndex = results.findIndex(function (el) { return url === el.url; }); return firstIndex === index; } function _liveFilter(result, index, results) { if (result.type !== 'live') return false; // filter duplicates var videoId = result.videoId; var firstIndex = results.findIndex(function (el) { return videoId === el.videoId; }); return firstIndex === index; } function _allFilter(result, index, results) { switch (result.type) { case 'video': case 'list': case 'channel': case 'live': break; default: // unsupported type return false; } // filter duplicates var url = result.url; var firstIndex = results.findIndex(function (el) { return url === el.url; }); return firstIndex === index; } /* Request search page results with provided * search_query term */ function getSearchResults(_options, callback) { // querystring variables var q = _querystring.escape(_options.search).split(/\s+/); var hl = _options.hl || 'en'; var gl = _options.gl || 'US'; var category = _options.category || ''; // music var pageStart = Number(_options.pageStart) || 1; var pageEnd = Number(_options.pageEnd) || Number(_options.pages) || 1; // handle zero-index start if (pageStart <= 0) { pageStart = 1; if (pageEnd >= 1) { pageEnd += 1; } } if (Number.isNaN(pageEnd)) { callback('error: pageEnd must be a number'); } _options.pageStart = pageStart; _options.pageEnd = pageEnd; _options.currentPage = _options.currentPage || pageStart; var queryString = '?'; queryString += 'search_query=' + q.join('+'); // language // queryString += '&' if (queryString.indexOf('&hl=') === -1) { queryString += '&hl=' + hl; } // location // queryString += '&' if (queryString.indexOf('&gl=') === -1) { queryString += '&gl=' + gl; } if (category) { // ex. "music" queryString += '&category=' + category; } if (_options.sp) { queryString += '&sp=' + _options.sp; } var uri = TEMPLATES.SEARCH_DESKTOP + queryString; var params = _url.parse(uri); params.headers = { 'user-agent': _userAgent, 'accept': 'text/html', 'accept-encoding': 'gzip', 'accept-language': 'en-US' }; debug(params); debug('getting results: ' + _options.currentPage); _dasu.req(params, function (err, res, body) { if (err) { callback(err); } else { if (res.status !== 200) { return callback('http status: ' + res.status); } if (_debugging) { var fs = require('fs'); var path = require('path'); fs.writeFileSync('dasu.response', res.responseText, 'utf8'); } try { _parseSearchResultInitialData(body, function (err, results) { if (err) return callback(err); var list = results; var videos = list.filter(_videoFilter); var playlists = list.filter(_playlistFilter); var channels = list.filter(_channelFilter); var live = list.filter(_liveFilter); var all = list.filter(_allFilter); // keep saving results into temporary memory while // we get more results _options._data = _options._data || {}; // init memory _options._data.videos = _options._data.videos || []; _options._data.playlists = _options._data.playlists || []; _options._data.channels = _options._data.channels || []; _options._data.live = _options._data.live || []; _options._data.all = _options._data.all || []; // push received results into memory videos.forEach(function (item) { _options._data.videos.push(item); }); playlists.forEach(function (item) { _options._data.playlists.push(item); }); channels.forEach(function (item) { _options._data.channels.push(item); }); live.forEach(function (item) { _options._data.live.push(item); }); all.forEach(function (item) { _options._data.all.push(item); }); _options.currentPage++; var getMoreResults = _options.currentPage <= _options.pageEnd; if (getMoreResults && results._sp) { _options.sp = results._sp; setTimeout(function () { getSearchResults(_options, callback); }, 2500); // delay a bit to try and prevent throttling } else { var _videos = _options._data.videos.filter(_videoFilter); var _playlists = _options._data.playlists.filter(_playlistFilter); var _channels = _options._data.channels.filter(_channelFilter); var _live = _options._data.live.filter(_liveFilter); var _all = _options._data.all.slice(_allFilter); // return all found videos callback(null, { all: _all, videos: _videos, live: _live, playlists: _playlists, lists: _playlists, accounts: _channels, channels: _channels }); } }); } catch (err) { callback(err); } } }); } /* For "modern" user-agents the html document returned from * YouTube contains initial json data that is used to populate * the page with JavaScript. This function will aim to find and * parse such data. */ function _parseSearchResultInitialData(responseText, callback) { var re = /{.*}/; var $ = _cheerio.load(responseText); var initialData = $('div#initial-data').html() || ''; initialData = re.exec(initialData) || ''; if (!initialData) { var scripts = $('script'); for (var i = 0; i < scripts.length; i++) { var script = $(scripts[i]).html(); var lines = script.split('\n'); lines.forEach(function (line) { var i; while ((i = line.indexOf('ytInitialData')) >= 0) { line = line.slice(i + 'ytInitialData'.length); var match = re.exec(line); if (match && match.length > initialData.length) { initialData = match; } } }); } } if (!initialData) { return callback('could not find inital data in the html document'); } var errors = []; var results = []; var json = JSON.parse(initialData[0]); var items = _jp.query(json, '$..itemSectionRenderer..contents.*'); // support newer richGridRenderer html structure _jp.query(json, '$..primaryContents..contents.*').forEach(function (item) { items.push(item); }); debug('items.length: ' + items.length); for (var _i = 0; _i < items.length; _i++) { var item = items[_i]; var result = undefined; var type = 'unknown'; var hasList = _jp.value(item, '$..compactPlaylistRenderer') || _jp.value(item, '$..playlistRenderer'); var hasChannel = _jp.value(item, '$..compactChannelRenderer') || _jp.value(item, '$..channelRenderer'); var hasVideo = _jp.value(item, '$..compactVideoRenderer') || _jp.value(item, '$..videoRenderer'); var listId = hasList && _jp.value(item, '$..playlistId'); var channelId = hasChannel && _jp.value(item, '$..channelId'); var videoId = hasVideo && _jp.value(item, '$..videoId'); var watchingLabel = _jp.query(item, '$..viewCountText..text').join(''); var isUpcoming = // if scheduled livestream (has not started yet) _jp.query(item, '$..thumbnailOverlayTimeStatusRenderer..style').join('').toUpperCase().trim() === 'UPCOMING'; var isLive = watchingLabel.indexOf('watching') >= 0 || _jp.query(item, '$..badges..label').join('').toUpperCase().trim() === 'LIVE NOW' || _jp.query(item, '$..thumbnailOverlayTimeStatusRenderer..text').join('').toUpperCase().trim() === 'LIVE' || isUpcoming; if (videoId) { type = 'video'; } if (channelId) { type = 'channel'; } if (listId) { type = 'list'; } if (isLive) { type = 'live'; } try { switch (type) { case 'video': { var thumbnail = _normalizeThumbnail(_jp.value(item, '$..thumbnail..url')) || _normalizeThumbnail(_jp.value(item, '$..thumbnails..url')) || _normalizeThumbnail(_jp.value(item, '$..thumbnails')); var title = _jp.value(item, '$..title..text') || _jp.value(item, '$..title..simpleText'); var author_name = _jp.value(item, '$..shortBylineText..text') || _jp.value(item, '$..longBylineText..text'); var author_url = _jp.value(item, '$..shortBylineText..url') || _jp.value(item, '$..longBylineText..url'); // publish/upload date var agoText = _jp.value(item, '$..publishedTimeText..text') || _jp.value(item, '$..publishedTimeText..simpleText'); var viewCountText = _jp.value(item, '$..viewCountText..text') || _jp.value(item, '$..viewCountText..simpleText') || "0"; var viewsCount = Number(viewCountText.split(/\s+/)[0].split(/[,.]/).join('').trim()); var lengthText = _jp.value(item, '$..lengthText..text') || _jp.value(item, '$..lengthText..simpleText'); var duration = _parseDuration(lengthText || '0:00'); var description = _jp.query(item, '$..detailedMetadataSnippets..snippetText..text').join('') || _jp.query(item, '$..description..text').join('') || _jp.query(item, '$..descriptionSnippet..text').join(''); // url ( playlist ) // const url = _jp.value( item, '$..navigationEndpoint..url' ) var url = TEMPLATES.YT + '/watch?v=' + videoId; result = { type: 'video', videoId: videoId, url: url, title: title.trim(), description: description, image: thumbnail, thumbnail: thumbnail, seconds: Number(duration.seconds), timestamp: duration.timestamp, duration: duration, ago: agoText, views: Number(viewsCount), author: { name: author_name, url: TEMPLATES.YT + author_url } }; } break; case 'list': { var _thumbnail = _normalizeThumbnail(_jp.value(item, '$..thumbnail..url')) || _normalizeThumbnail(_jp.value(item, '$..thumbnails..url')) || _normalizeThumbnail(_jp.value(item, '$..thumbnails')); var _title = _jp.value(item, '$..title..text') || _jp.value(item, '$..title..simpleText'); var _author_name = _jp.value(item, '$..shortBylineText..text') || _jp.value(item, '$..longBylineText..text') || _jp.value(item, '$..shortBylineText..simpleText') || _jp.value(item, '$..longBylineText..simpleTextn') || 'YouTube'; var _author_url = _jp.value(item, '$..shortBylineText..url') || _jp.value(item, '$..longBylineText..url') || ''; var video_count = _jp.value(item, '$..videoCountShortText..text') || _jp.value(item, '$..videoCountText..text') || _jp.value(item, '$..videoCountShortText..simpleText') || _jp.value(item, '$..videoCountText..simpleText') || _jp.value(item, '$..thumbnailText..text') || _jp.value(item, '$..thumbnailText..simpleText'); // url ( playlist ) // const url = _jp.value( item, '$..navigationEndpoint..url' ) var _url2 = TEMPLATES.YT + '/playlist?list=' + listId; result = { type: 'list', listId: listId, url: _url2, title: _title.trim(), image: _thumbnail, thumbnail: _thumbnail, videoCount: video_count, author: { name: _author_name, url: TEMPLATES.YT + _author_url } }; } break; case 'channel': { var _thumbnail2 = _normalizeThumbnail(_jp.value(item, '$..thumbnail..url')) || _normalizeThumbnail(_jp.value(item, '$..thumbnails..url')) || _normalizeThumbnail(_jp.value(item, '$..thumbnails')); var _title2 = _jp.value(item, '$..title..text') || _jp.value(item, '$..title..simpleText') || _jp.value(item, '$..displayName..text'); var _author_name2 = _jp.value(item, '$..shortBylineText..text') || _jp.value(item, '$..longBylineText..text') || _jp.value(item, '$..displayName..text') || _jp.value(item, '$..displayName..simpleText'); var video_count_label = _jp.value(item, '$..videoCountText..text') || _jp.value(item, '$..videoCountText..simpleText') || '0'; var sub_count_label = _jp.value(item, '$..subscriberCountText..text') || _jp.value(item, '$..subscriberCountText..simpleText'); // first space separated word that has digits if (typeof sub_count_label === 'string') { sub_count_label = sub_count_label.split(/\s+/).filter(function (w) { return w.match(/\d/); })[0]; } // url ( playlist ) // const url = _jp.value( item, '$..navigationEndpoint..url' ) var _url3 = _jp.value(item, '$..navigationEndpoint..url') || '/user/' + _title2; result = { type: 'channel', name: _author_name2, url: TEMPLATES.YT + _url3, title: _title2.trim(), image: _thumbnail2, thumbnail: _thumbnail2, videoCount: Number(video_count_label.replace(/\D+/g, '')), videoCountLabel: video_count_label, subCount: _parseSubCountLabel(sub_count_label), subCountLabel: sub_count_label }; } break; case 'live': { var _thumbnail3 = _normalizeThumbnail(_jp.value(item, '$..thumbnail..url')) || _normalizeThumbnail(_jp.value(item, '$..thumbnails..url')) || _normalizeThumbnail(_jp.value(item, '$..thumbnails')); var _title3 = _jp.value(item, '$..title..text') || _jp.value(item, '$..title..simpleText'); var _author_name3 = _jp.value(item, '$..shortBylineText..text') || _jp.value(item, '$..longBylineText..text'); var _author_url2 = _jp.value(item, '$..shortBylineText..url') || _jp.value(item, '$..longBylineText..url'); var _watchingLabel = _jp.query(item, '$..viewCountText..text').join('') || _jp.query(item, '$..viewCountText..simpleText').join('') || '0'; var watchCount = Number(_watchingLabel.split(/\s+/)[0].split(/[,.]/).join('').trim()); var _description = _jp.query(item, '$..detailedMetadataSnippets..snippetText..text').join('') || _jp.query(item, '$..description..text').join('') || _jp.query(item, '$..descriptionSnippet..text').join(''); var scheduledEpochTime = _jp.value(item, '$..upcomingEventData..startTime'); var scheduledTime = Date.now() > scheduledEpochTime ? scheduledEpochTime * 1000 : scheduledEpochTime; var scheduledDateString = _toInternalDateString(scheduledTime); // url ( playlist ) // const url = _jp.value( item, '$..navigationEndpoint..url' ) var _url4 = TEMPLATES.YT + '/watch?v=' + videoId; result = { type: 'live', videoId: videoId, url: _url4, title: _title3.trim(), description: _description, image: _thumbnail3, thumbnail: _thumbnail3, watching: Number(watchCount), author: { name: _author_name3, url: TEMPLATES.YT + _author_url2 } }; if (scheduledTime) { result.startTime = scheduledTime; result.startDate = scheduledDateString; result.status = 'UPCOMING'; } else { result.status = 'LIVE'; } } break; default: // ignore other stuff } if (result) { results.push(result); } } catch (err) { debug(err); errors.push(err); } } var ctoken = _jp.value(json, '$..continuation'); results._ctoken = ctoken; if (errors.length) { return callback(errors.pop(), results); } return callback(null, results); } /* Get metadata of a single video */ function getVideoMetaData(opts, callback) { debug('fn: getVideoMetaData'); var videoId; if (typeof opts === 'string') { videoId = opts; } if (_typeof(opts) === 'object') { videoId = opts.videoId; } var _opts$hl = opts.hl, hl = _opts$hl === void 0 ? 'en' : _opts$hl, _opts$gl = opts.gl, gl = _opts$gl === void 0 ? 'US' : _opts$gl; var uri = "https://www.youtube.com/watch?hl=".concat(hl, "&gl=").concat(gl, "&v=").concat(videoId); var params = _url.parse(uri); params.headers = { 'user-agent': _userAgent, 'accept': 'text/html', 'accept-encoding': 'gzip', 'accept-language': "".concat(hl, "-").concat(gl) }; params.headers['user-agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15'; _dasu.req(params, function (err, res, body) { if (err) { callback(err); } else { if (res.status !== 200) { return callback('http status: ' + res.status); } if (_debugging) { var fs = require('fs'); var path = require('path'); fs.writeFileSync('dasu.response', res.responseText, 'utf8'); } try { _parseVideoInitialData(body, callback); } catch (err) { callback(err); } } }); } function _parseVideoInitialData(responseText, callback) { debug('_parseVideoInitialData'); // const fs = require( 'fs' ) // fs.writeFileSync( 'tmp.file', responseText ) responseText = _getScripts(responseText); var initialData = _between(_findLine(/ytInitialData.*=\s*{/, responseText), '{', '}'); if (!initialData) { return callback('could not find inital data in the html document'); } var initialPlayerData = _between(_findLine(/ytInitialPlayerResponse.*=\s*{/, responseText), '{', '}'); if (!initialPlayerData) { return callback('could not find inital player data in the html document'); } // debug( initialData[ 0 ] ) // debug( '\n------------------\n' ) // debug( initialPlayerData[ 0 ] ) var idata = JSON.parse(initialData); var ipdata = JSON.parse(initialPlayerData); var videoId = _jp.value(idata, '$..currentVideoEndpoint..videoId'); if (!videoId) { return callback('video unavailable'); } if (_jp.value(ipdata, '$..status') === 'ERROR' || _jp.value(ipdata, '$..reason') === 'Video unavailable') { return callback('video unavailable'); } var title = _parseVideoMeataDataTitle(idata); var description = _jp.query(idata, '$..description..text').join('') || _jp.query(ipdata, '$..description..simpleText').join('') || _jp.query(ipdata, '$..microformat..description..simpleText').join('') || _jp.query(ipdata, '$..videoDetails..shortDescription').join(''); var author_name = _jp.value(idata, '$..owner..title..text') || _jp.value(idata, '$..owner..title..simpleText'); var author_url = _jp.value(idata, '$..owner..navigationEndpoint..url') || _jp.value(idata, '$..owner..title..url'); var thumbnailUrl = 'https://i.ytimg.com/vi/' + videoId + '/hqdefault.jpg'; var seconds = Number(_jp.value(ipdata, '$..videoDetails..lengthSeconds')); var timestamp = _msToTimestamp(seconds * 1000); var duration = _parseDuration(timestamp); // TODO some video's have likes/dislike ratio hidden (ex: 62ezXENOuIA) // which makes this value undefined // const sentimentBar = ( // // ex. "tooltip": "116,701 / 8,930" // _jp.value( idata, '$..sentimentBar..tooltip' ) // .split( /[,.]/ ).join( '' ) // .split( /\D+/ ) // ) // // TODO currently not in use // const likes = Number( sentimentBar[ 0 ] ) // const dislikes = Number( sentimentBar[ 1 ] ) var uploadDate = _jp.value(idata, '$..uploadDate') || _jp.value(idata, '$..dateText..simpleText'); var agoText = uploadDate && _humanTime(new Date(uploadDate)) || ''; var video = { title: title, description: description, url: TEMPLATES.YT + '/watch?v=' + videoId, videoId: videoId, seconds: Number(duration.seconds), timestamp: duration.timestamp, duration: duration, views: Number(_jp.value(ipdata, '$..videoDetails..viewCount')), genre: (_jp.value(ipdata, '$..category') || '').toLowerCase(), uploadDate: _toInternalDateString(uploadDate), ago: agoText, // ex: 10 years ago image: thumbnailUrl, thumbnail: thumbnailUrl, author: { name: author_name, url: TEMPLATES.YT + author_url } }; callback(null, video); } /* Get metadata from a playlist page */ function getPlaylistMetaData(opts, callback) { debug('fn: getPlaylistMetaData'); var listId; if (typeof opts === 'string') { listId = opts; } if (_typeof(opts) === 'object') { listId = opts.listId || opts.playlistId; } var _opts$hl2 = opts.hl, hl = _opts$hl2 === void 0 ? 'en' : _opts$hl2, _opts$gl2 = opts.gl, gl = _opts$gl2 === void 0 ? 'US' : _opts$gl2; var uri = "https://www.youtube.com/playlist?hl=".concat(hl, "&gl=").concat(gl, "&list=").concat(listId); var params = _url.parse(uri); params.headers = { 'user-agent': _userAgent, 'accept': 'text/html', 'accept-encoding': 'gzip', 'accept-language': "".concat(hl, "-").concat(gl) }; _dasu.req(params, function (err, res, body) { if (err) { callback(err); } else { if (res.status !== 200) { return callback('http status: ' + res.status); } if (_debugging) { var fs = require('fs'); var path = require('path'); fs.writeFileSync('dasu.response', res.responseText, 'utf8'); } try { _parsePlaylistInitialData(body, callback); } catch (err) { callback(err); } } }); } function _parsePlaylistInitialData(responseText, callback) { debug('fn: parsePlaylistBody'); responseText = _getScripts(responseText); var jsonString = responseText.match(/ytInitialData.*=\s*({.*});/)[1]; // console.log( jsonString ) if (!jsonString) { throw new Error('failed to parse ytInitialData json data'); } var json = JSON.parse(jsonString); //console.log( json ) // check for errors (ex: noexist/unviewable playlist) var plerr = _jp.value(json, '$..alerts..alertRenderer'); if (plerr && typeof plerr.type === 'string' && plerr.type.toLowerCase() === 'error') { var plerrtext = 'playlist error, not found?'; if (_typeof(plerr.text) === 'object') { plerrtext = _jp.query(plerr.text, '$..text').join(''); } if (typeof plerr.text === 'string') { plerrtext = plerr.text; } throw new Error('playlist error: ' + plerrtext); } var alertInfo = ''; _jp.query(json, '$..alerts..text').forEach(function (val) { if (typeof val === 'string') alertInfo += val; if (_typeof(val) === 'object') { // try grab simpletex var simpleText = _jp.value(val, '$..simpleText'); if (simpleText) alertInfo += simpleText; } }); var listId = _jp.value(json, '$..microformat..urlCanonical').split('=')[1]; // console.log( 'listId: ' + listId ) var viewCount = 0; try { var viewCountLabel = _jp.value(json, '$..sidebar.playlistSidebarRenderer.items[0]..stats[1].simpleText'); if (viewCountLabel.toLowerCase() === 'no views') { viewCount = 0; } else { viewCount = viewCountLabel.match(/\d+/g).join(''); } } catch (err) { /* ignore */ } var size = (_jp.value(json, '$..sidebar.playlistSidebarRenderer.items[0]..stats[0].simpleText') || _jp.query(json, '$..sidebar.playlistSidebarRenderer.items[0]..stats[0]..text').join('')).match(/\d+/g).join(''); // playlistVideoListRenderer contents var list = _jp.query(json, '$..playlistVideoListRenderer..contents')[0]; // TODO unused atm var listHasContinuation = _typeof(list[list.length - 1].continuationItemRenderer) === 'object'; // const list = _jp.query( json, '$..contents..tabs[0]..contents[0]..contents[0]..contents' )[ 0 ] var videos = []; list.forEach(function (item) { if (!item.playlistVideoRenderer) return; // skip var json = item; var duration = _parseDuration(_jp.value(json, '$..lengthText..simpleText') || _jp.value(json, '$..thumbnailOverlayTimeStatusRenderer..simpleText') || _jp.query(json, '$..lengthText..text').join('') || _jp.query(json, '$..thumbnailOverlayTimeStatusRenderer..text').join('')); var video = { title: _jp.value(json, '$..title..simpleText') || _jp.value(json, '$..title..text') || _jp.query(json, '$..title..text').join(''), videoId: _jp.value(json, '$..videoId'), listId: listId, thumbnail: _normalizeThumbnail(_jp.value(json, '$..thumbnail..url')) || _normalizeThumbnail(_jp.value(json, '$..thumbnails..url')) || _normalizeThumbnail(_jp.value(json, '$..thumbnails')), // ref: issue #35 https://github.com/talmobi/yt-search/issues/35 duration: duration, author: { name: _jp.value(json, '$..shortBylineText..runs[0]..text'), url: 'https://youtube.com' + _jp.value(json, '$..shortBylineText..runs[0]..url') } }; videos.push(video); }); // console.log( videos ) // console.log( 'videos.length: ' + videos.length ) var plthumbnail = _normalizeThumbnail(_jp.value(json, '$..microformat..thumbnail..url')) || _normalizeThumbnail(_jp.value(json, '$..microformat..thumbnails..url')) || _normalizeThumbnail(_jp.value(json, '$..microformat..thumbnails')); var playlist = { title: _jp.value(json, '$..microformat..title'), listId: listId, url: 'https://youtube.com/playlist?list=' + listId, size: Number(size), views: Number(viewCount), // lastUpdate: lastUpdate, date: _parsePlaylistLastUpdateTime(_jp.value(json, '$..sidebar.playlistSidebarRenderer.items[0]..stats[2]..simpleText') || _jp.query(json, '$..sidebar.playlistSidebarRenderer.items[0]..stats[2]..text').join('') || ''), image: plthumbnail || videos[0].thumbnail, thumbnail: plthumbnail || videos[0].thumbnail, // playlist items/videos videos: videos, alertInfo: alertInfo, author: { name: _jp.value(json, '$..videoOwner..title..runs[0]..text'), url: 'https://youtube.com' + _jp.value(json, '$..videoOwner..navigationEndpoint..url') } }; callback && callback(null, playlist); } function _parsePlaylistLastUpdateTime(lastUpdateLabel) { debug('fn: _parsePlaylistLastUpdateTime'); var DAY_IN_MS = 1000 * 60 * 60 * 24; try { // ex "Last Updated on Jun 25, 2018" // ex: "Viimeksi päivitetty 25.6.2018" var words = lastUpdateLabel.toLowerCase().trim().split(/[\s.-]+/); if (words.length > 0) { var lastWord = words[words.length - 1].toLowerCase(); if (lastWord === 'yesterday') { var ms = Date.now() - DAY_IN_MS; var d = new Date(ms); // a day earlier than today if (d.toString() !== 'Invalid Date') return _toInternalDateString(d); } } if (words.length >= 2) { // handle strings like "7 days ago" if (words[0] === 'updated' && words[2].slice(0, 3) === 'day') { var _ms = Date.now() - DAY_IN_MS * words[1]; var _d = new Date(_ms); // a day earlier than today if (_d.toString() !== 'Invalid Date') return _toInternalDateString(_d); } } for (var i = 0; i < words.length; i++) { var slice = words.slice(i); var t = slice.join(' '); var r = slice.reverse().join(' '); var _a = new Date(t); var b = new Date(r); if (_a.toString() !== 'Invalid Date') return _toInternalDateString(_a); if (b.toString() !== 'Invalid Date') return _toInternalDateString(b); } return ''; } catch (err) { return ''; } } function _toInternalDateString(date) { date = new Date(date); debug('fn: _toInternalDateString'); return date.getFullYear() + '-' + (date.getMonth() + 1) + '-' + // january gives 0 date.getDate(); } /* Helper fn to parse duration labels * ex: Duration: 2:27, Kesto: 1.07.54 */ function _parseDuration(timestampText) { var a = timestampText.split(/\s+/); var lastword = a[a.length - 1]; // ex: Duration: 2:27, Kesto: 1.07.54 // replace all non :, non digits and non . var timestamp = lastword.replace(/[^:.\d]/g, ''); if (!timestamp) return { toString: function toString() { return a[0]; }, seconds: 0, timestamp: 0 }; // remove trailing junk that are not digits while (timestamp[timestamp.length - 1].match(/\D/)) { timestamp = timestamp.slice(0, -1); } // replaces all dots with nice ':' timestamp = timestamp.replace(/\./g, ':'); var t = timestamp.split(/[:.]/); var seconds = 0; var exp = 0; for (var i = t.length - 1; i >= 0; i--) { if (t[i].length <= 0) continue; var number = t[i].replace(/\D/g, ''); // var exp = (t.length - 1) - i; seconds += parseInt(number) * (exp > 0 ? Math.pow(60, exp) : 1); exp++; if (exp > 2) break; } ; return { toString: function toString() { return seconds + ' seconds (' + timestamp + ')'; }, seconds: seconds, timestamp: timestamp }; } /* Parses a type of human-like timestamps found on YouTube. * ex: "PT4M13S" -> "4:13" */ function _parseHumanDuration(timestampText) { debug('_parseHumanDuration'); // ex: PT4M13S var pt = timestampText.slice(0, 2); var timestamp = timestampText.slice(2).toUpperCase(); if (pt !== 'PT') return { toString: function toString() { return a[0]; }, seconds: 0, timestamp: 0 }; var h = timestamp.match(/\d?\dH/); var m = timestamp.match(/\d?\dM/); var s = timestamp.match(/\d?\dS/); h = h && h[0].slice(0, -1) || 0; m = m && m[0].slice(0, -1) || 0; s = s && s[0].slice(0, -1) || 0; h = parseInt(h); m = parseInt(m); s = parseInt(s); timestamp = ''; if (h) timestamp += h + ':'; if (m) timestamp += m + ':'; timestamp += s; var seconds = h * 60 * 60 + m * 60 + s; return { toString: function toString() { return seconds + ' seconds (' + timestamp + ')'; }, seconds: seconds, timestamp: timestamp }; } /* Helper fn to parse sub count labels * and turn them into Numbers. * * It's an estimate but can be useful for sorting etc. * * ex. "102M subscribers" -> 102000000 * ex. "5.33m subscribers" -> 5330000 */ function _parseSubCountLabel(subCountLabel) { if (!subCountLabel) return undefined; var label = subCountLabel.split(/\s+/).filter(function (w) { return w.match(/\d/); })[0].toLowerCase(); var m = label.match(/\d+(\.\d+)?/); if (m && m[0]) {} else { return; } var num = Number(m[0]); var THOUSAND = 1000; var MILLION = THOUSAND * THOUSAND; if (label.indexOf('m') >= 0) return MILLION * num; if (label.indexOf('k') >= 0) return THOUSAND * num; return num; } /* Helper fn to choose a good thumbnail. */ function _normalizeThumbnail(thumbnails) { var t; if (typeof thumbnails === 'string') { t = thumbnails; } else { // handle as array if (thumbnails.length) { t = thumbnails[0]; return _normalizeThumbnail(t); } // failed to parse thumbnail return undefined; } t = t.split('?')[0]; t = t.split('/default.jpg').join('/hqdefault.jpg'); t = t.split('/default.jpeg').join('/hqdefault.jpeg'); if (t.indexOf('//') === 0) { return 'https://' + t.slice(2); } return t.split('http://').join('https://'); } /* Helper fn to transform ms to timestamp * ex: 253000 -> "4:13" */ function _msToTimestamp(ms) { var t = ''; var MS_HOUR = 1000 * 60 * 60; var MS_MINUTE = 1000 * 60; var MS_SECOND = 1000; var h = Math.floor(ms / MS_HOUR); var m = Math.floor(ms / MS_MINUTE) % 60; var s = Math.floor(ms / MS_SECOND) % 60; if (h) t += h + ':'; // pad with extra zero only if hours are set if (h && String(m).length < 2) t += '0'; t += m + ':'; // pad with extra zero if (String(s).length < 2) t += '0'; t += s; return t; } function _parseVideoMeataDataTitle(idata) { var t = _jp.query(idata, '$..videoPrimaryInfoRenderer.title..text').join('') || _jp.query(idata, '$..videoPrimaryInfoRenderer.title..simpleText').join('') || _jp.query(idata, '$..videoPrimaryRenderer.title..text').join('') || _jp.query(idata, '$..videoPrimaryRenderer.title..simpleText').join('') || _jp.value(idata, '$..title..text') || _jp.value(idata, '$..title..simpleText'); // remove zero-width chars return t.replace(/[\u0000-\u001F\u007F-\u009F\u200b]/g, ''); } // run tests is script is run directly if (require.main === module) { // https://www.youtube.com/watch?v=e9vrfEoc8_g test('superman theme list pewdiepie channel'); } function test(query) { console.log('test: doing list search'); var opts = { query: query, pageEnd: 1 }; search(opts, function (error, r) { if (error) throw error; var videos = r.videos; var playlists = r.playlists; var channels = r.channels; console.log('videos: ' + videos.length); console.log('playlists: ' + playlists.length); console.log('channels: ' + channels.length); for (var i = 0; i < videos.length; i++) { var song = videos[i]; var time = " (".concat(song.timestamp, ")"); console.log(song.title + time); } playlists.forEach(function (p) { console.log("playlist: ".concat(p.title, " | ").concat(p.listId)); }); channels.forEach(function (c) { console.log("channel: ".concat(c.title, " | ").concat(c.description)); }); }); } },{"./util.js":2,"async.parallellimit":undefined,"cheerio":undefined,"dasu":undefined,"fs":undefined,"human-time":undefined,"jsonpath-plus":undefined,"path":undefined,"querystring":undefined,"url":undefined}],2:[function(require,module,exports){ "use strict"; var _cheerio = require('cheerio'); var util = {}; module.exports = util; util._getScripts = _getScripts; util._findLine = _findLine; util._between = _between; function _getScripts(text) { // match all contents within html script tags var $ = _cheerio.load(text); var scripts = $('script'); var buffer = ''; // combine all scripts for (var i = 0; i < scripts.length; i++) { var el = scripts[i]; var child = el && el.children[0]; var data = child && child.data; if (data) { buffer += data + '\n'; } } return buffer; } function _findLine(regex, text) { var cache = _findLine.cache || {}; _findLine.cache = cache; cache[text] = cache[text] || {}; var lines = cache[text].lines || text.split('\n'); cache[text].lines = lines; clearTimeout(cache[text].timeout); cache[text].timeout = setTimeout(function () { delete cache[text]; }, 100); for (var i = 0; i < lines.length; i++) { var line = lines[i]; if (regex.test(line)) return line; } return ''; } function _between(text, start, end) { var i = text.indexOf(start); var j = text.lastIndexOf(end); if (i < 0) return ''; if (j < 0) return ''; return text.slice(i, j + 1); } },{"cheerio":undefined}]},{},[1])(1) });