import { program } from 'commander' ; import { accessSync , constants } from 'fs' ; import { remove } from 'fs-extra' ; import { join } from 'path' ; import { YoutubeDLCLI , YoutubeDLInfo , YoutubeDLInfoBuilder } from '@server/helpers/youtube-dl' ; import { wait } from '@shared/core-utils' ; import { sha256 } from '@shared/extra-utils' ; import { doRequestAndSaveToFile } from '../helpers/requests' ; import { assignToken , buildCommonVideoOptions , buildServer , buildVideoAttributesFromCommander , getLogger , getServerCredentials } from './cli' ; import prompt = require ( 'prompt' ) ; const processOptions = { maxBuffer : Infinity } ; let command = program . name ( 'import-videos' ) ; command = buildCommonVideoOptions ( command ) ; command . option ( '-u, --url <url>' , 'Server url' ) . option ( '-U, --username <username>' , 'Username' ) . option ( '-p, --password <token>' , 'Password' ) . option ( '--target-url <targetUrl>' , 'Video target URL' ) . option ( '--since <since>' , 'Publication date (inclusive) since which the videos can be imported (YYYY-MM-DD)' , parseDate ) . option ( '--until <until>' , 'Publication date (inclusive) until which the videos can be imported (YYYY-MM-DD)' , parseDate ) . option ( '--first <first>' , 'Process first n elements of returned playlist' ) . option ( '--last <last>' , 'Process last n elements of returned playlist' ) . option ( '--wait-interval <waitInterval>' , 'Duration between two video imports (in seconds)' , convertIntoMs ) . option ( '-T, --tmpdir <tmpdir>' , 'Working directory' , __dirname ) . usage ( "[global options] [ -- youtube-dl options]" ) . parse ( process . argv ) ; const options = command . opts ( ) ; const log = getLogger ( options . verbose ) ; getServerCredentials ( command ) . then ( ( { url , username , password } ) => { if ( ! options . targetUrl ) { exitError ( '--target-url field is required.' ) ; } try { accessSync ( options . tmpdir , constants . R_OK | constants . W_OK ) ; } catch ( e ) { exitError ( '--tmpdir %s: directory does not exist or is not accessible' , options . tmpdir ) ; } url = normalizeTargetUrl ( url ) ; options . targetUrl = normalizeTargetUrl ( options . targetUrl ) ; run ( url , username , password ) . catch ( err => exitError ( err ) ) ; } ) . catch ( err => console . error ( err ) ) ; async function run ( url , username , password ) { if ( ! password ) password = await promptPassword ( ) ; const youtubeDLBinary = await YoutubeDLCLI . safeGet ( ) ; let info = await getYoutubeDLInfo ( youtubeDLBinary , options . targetUrl , command . args ) ; if ( ! Array . isArray ( info ) ) info = [ info ] ; const uploadsObject = info . find ( i => ! i . ie_key && ! i . duration && i . title === 'Uploads' ) ; if ( uploadsObject ) { console . log ( 'Fixing URL to %s.' , uploadsObject . url ) ; info = await getYoutubeDLInfo ( youtubeDLBinary , uploadsObject . url , command . args ) ; } let infoArray ; infoArray = [ ] . concat ( info ) ; if ( options . first ) { infoArray = infoArray . slice ( 0 , options . first ) ; } else if ( options . last ) { infoArray = infoArray . slice ( - options . last ) ; } log . info ( 'Will download and upload %d videos.\n' , infoArray . length ) ; let skipInterval = true ; for ( const [ index , info ] of infoArray . entries ( ) ) { try { if ( index > 0 && options . waitInterval && ! skipInterval ) { log . info ( "Wait for %d seconds before continuing." , options . waitInterval / 1000 ) ; await wait ( options . waitInterval ) ; } skipInterval = await processVideo ( { cwd : options . tmpdir , url , username , password , youtubeInfo : info } ) ; } catch ( err ) { console . error ( 'Cannot process video.' , { info , url , err } ) ; } } log . info ( 'Video/s for user %s imported: %s' , username , options . targetUrl ) ; process . exit ( 0 ) ; } async function processVideo ( parameters ) { const { youtubeInfo , cwd , url , username , password } = parameters ; log . debug ( 'Fetching object.' , youtubeInfo ) ; const videoInfo = await fetchObject ( youtubeInfo ) ; log . debug ( 'Fetched object.' , videoInfo ) ; if ( options . since && videoInfo . originallyPublishedAt && videoInfo . originallyPublishedAt . getTime ( ) < options . since . getTime ( ) ) { log . info ( 'Video "%s" has been published before "%s", don\'t upload it.\n' , videoInfo . name , formatDate ( options . since ) ) ; return true ; } if ( options . until && videoInfo . originallyPublishedAt && videoInfo . originallyPublishedAt . getTime ( ) > options . until . getTime ( ) ) { log . info ( 'Video "%s" has been published after "%s", don\'t upload it.\n' , videoInfo . name , formatDate ( options . until ) ) ; return true ; } const server = buildServer ( url ) ; const { data } = await server . search . advancedVideoSearch ( { search : { search : videoInfo . name , sort : '-match' , searchTarget : 'local' } } ) ; log . info ( '############################################################\n' ) ; if ( data . find ( v => v . name === videoInfo . name ) ) { log . info ( 'Video "%s" already exists, don\'t reupload it.\n' , videoInfo . name ) ; return true ; } const path = join ( cwd , sha256 ( videoInfo . url ) + '.mp4' ) ; log . info ( 'Downloading video "%s"...' , videoInfo . name ) ; try { const youtubeDLBinary = await YoutubeDLCLI . safeGet ( ) ; const output = await youtubeDLBinary . download ( { url : videoInfo . url , format : YoutubeDLCLI . getYoutubeDLVideoFormat ( [ ] ) , output : path , additionalYoutubeDLArgs : command . args , processOptions } ) ; log . info ( output . join ( '\n' ) ) ; await uploadVideoOnPeerTube ( { cwd , url , username , password , videoInfo , videoPath : path } ) ; } catch ( err ) { log . error ( err . message ) ; } return false ; } async function uploadVideoOnPeerTube ( parameters ) { const { videoInfo , videoPath , cwd , url , username , password } = parameters ; const server = buildServer ( url ) ; await assignToken ( server , username , password ) ; let thumbnailfile ; if ( videoInfo . thumbnailUrl ) { thumbnailfile = join ( cwd , sha256 ( videoInfo . thumbnailUrl ) + '.jpg' ) ; await doRequestAndSaveToFile ( videoInfo . thumbnailUrl , thumbnailfile ) ; } const baseAttributes = await buildVideoAttributesFromCommander ( server , program , videoInfo ) ; const attributes = { ... baseAttributes , originallyPublishedAt : videoInfo . originallyPublishedAt ? videoInfo . originallyPublishedAt . toISOString ( ) : null , thumbnailfile , previewfile : thumbnailfile , fixture : videoPath } ; log . info ( '\nUploading on PeerTube video "%s".' , attributes . name ) ; try { await server . videos . upload ( { attributes } ) ; } catch ( err ) { if ( err . message . indexOf ( '401' ) !== - 1 ) { log . info ( 'Got 401 Unauthorized, token may have expired, renewing token and retry.' ) ; server . accessToken = await server . login . getAccessToken ( username , password ) ; await server . videos . upload ( { attributes } ) ; } else { exitError ( err . message ) ; } } await remove ( videoPath ) ; if ( thumbnailfile ) await remove ( thumbnailfile ) ; log . info ( 'Uploaded video "%s"!\n' , attributes . name ) ; } async function fetchObject ( info ) { const url = buildUrl ( info ) ; const youtubeDLCLI = await YoutubeDLCLI . safeGet ( ) ; const result = await youtubeDLCLI . getInfo ( { url , format : YoutubeDLCLI . getYoutubeDLVideoFormat ( [ ] ) , processOptions } ) ; const builder = new YoutubeDLInfoBuilder ( result ) ; const videoInfo = builder . getInfo ( ) ; return { ... videoInfo , url } ; } function buildUrl ( info ) { const webpageUrl = info . webpage_url as string ; if ( webpageUrl ?. match ( /^https?:\/\// ) ) return webpageUrl ; const url = info . url as string ; if ( url ?. match ( /^https?:\/\// ) ) return url ; return 'https://www.youtube.com/watch?v=' + info . id ; } function normalizeTargetUrl ( url ) { let normalizedUrl = url . replace ( /\/+$/ , '' ) ; if ( ! normalizedUrl . startsWith ( 'http://' ) && ! normalizedUrl . startsWith ( 'https://' ) ) { normalizedUrl = 'https://' + normalizedUrl ; } return normalizedUrl ; } async function promptPassword ( ) { return new Promise < string > ( ( res , rej ) => { prompt . start ( ) ; const schema = { properties : { password : { hidden : true , required : true } } } ; prompt . get ( schema , function ( err , result ) { if ( err ) { return rej ( err ) ; } return res ( result . password ) ; } ) ; } ) ; } function parseDate ( dateAsStr ) { if ( ! /\d{4}-\d{2}-\d{2}/ . test ( dateAsStr ) ) { exitError ( `Invalid date passed: ${ dateAsStr }. Expected format: YYYY-MM-DD. See help for usage.` ) ; } const date = new Date ( dateAsStr ) ; date . setHours ( 0 , 0 , 0 ) ; if ( isNaN ( date . getTime ( ) ) ) { exitError ( `Invalid date passed: ${ dateAsStr }. See help for usage.` ) ; } return date ; } function formatDate ( date ) { return date . toISOString ( ) . split ( 'T' ) [ 0 ] ; } function convertIntoMs ( secondsAsStr ) { const seconds = parseInt ( secondsAsStr , 10 ) ; if ( seconds <= 0 ) { exitError ( `Invalid duration passed: ${ seconds }. Expected duration to be strictly positive and in seconds` ) ; } return Math . round ( seconds * 1000 ) ; } function exitError ( message , ... meta ) { console . error ( message , ... meta ) ; process . exit ( - 1 ) ; } function getYoutubeDLInfo ( youtubeDLCLI , url , args ) { return youtubeDLCLI . getInfo ( { url , format : YoutubeDLCLI . getYoutubeDLVideoFormat ( [ ] ) , additionalYoutubeDLArgs : [ '-j' , '--flat-playlist' , '--playlist-reverse' , ... args ] , processOptions } ) ; }