launch.js


const path = require('path');
const fs = require('fs');

const Promise = require('bluebird');
const { Provider } = require('nconf');
const winston = require('winston');

const { TumblrImageDownloader } = require('./');

const pkg = JSON.parse(fs.readFileSync(path.join(__dirname, "..", "package.json"), 'utf8'));

Promise.promisifyAll(fs);

var logs, nconf;

/**
 * The environment variables that can be used to configure the application.
 * @type {string[]}
 * @constant
 * @default
 */
const env_whitelist = [
    "LOG_LEVEL",
    "TUMBLR_USERNAME",
    "TUMBLR_PASSWORD",
    "USERNAME",
    "PASSWORD",
    "PROXY_URL",
    "HTTP_PROXY"
];

/**
 * Converts a configuration property's name from env variable format to application config format
 * `"CONTROL_HOST"` -> `"controlHost"` 
 * @param {string} env - Environment variable
 * @returns {string}
 * @private
 */
function env_to_config(env) {
	let a = env.toLowerCase().split('_');
	i = 1;
	while (i < a.length) {
		a[i] = a[i][0].toUpperCase() + a[i].substr(1);
		i++;
	}
	return a.join('');
 }

/**
 * Creates an {@link TumblrImageDownloader} instance with the application configuration.
 * @param {Provider} nconf - Nconf instance to use.
 * @returns {TumblrImageDownloader}
 * @private
 */
function TIDFactory(nconf) {
    return new TumblrImageDownloader({
        proxy_url: nconf.get('proxy_url')
    });
}

/**
 * Returns the path to a {@link Photo} within a parent directory.
 * @param {string} parentDir - The parent directory the photos are contained in.
 * @param {Photo} photo - The photo to extra data from.
 * @returns {string} - The path of the photo.
 */
function getPhotoPath(parentDir, photo) {
    let ext = photo.photoUrl.split('/').slice(-1)[0].split('.').pop();
    let photoIdWithExt = `${photo.photoId}.${ext}`;
    return path.join(parentDir, photoIdWithExt);
}

/**
 * Downloads a blog saving it to a directory.
 * @param {Provider} nconf - The nconf instance.
 * @param {Logger} logs - The winston logger.
 * @async
 * @returns {Promise<number>} - Returns the exit code.
 */
async function download(argv, nconf, logs) {
    let dir = argv[1];
    let blogSubdomain = argv[0];
    if (!dir || !blogSubdomain) {
        logs.error('No username or directory provided');
        return 1;
    }
    let downloader = TIDFactory(nconf);
    let username = nconf.get('username');
    let password = nconf.get('password');
    let pageNumber = nconf.get('pageNumber');
    let stopAtIndex = nconf.get('stopAtIndex');
    let stopAtPage = nconf.get('stopAtPage');

    return new Promise((resolve, reject) => {
        let p;
        downloader.once('error', (error) => {
            logs.error(`error downloading from blog ${blogSubdomain}: ${error.message}`);
            p.cancel();
            resolve(1);
        });
        
        downloader.on('photo', (async (photo) => {
            let photoPath = getPhotoPath(dir, photo);
            try {
                await fs.writeFileAsync(photoPath, photo.photoBytes);
                logs.verbose(`saved "${photo.photoId}" to "${photoPath}"`);
            } catch (error) {
                logs.error(`error saving photo "${photo.photoId}" to ${photoPath}`);
                resolve(1);
            }
        }));

        downloader.on('pageChange', (page_info) => {
            logs.info(`downloading page "${page_info.pageNumber}" of "${page_info.blogSubdomain}"`);
        });

        p = (async () => {
            try {
                if (!fs.existsSync(dir))
                    await fs.mkdirAsync(dir);
                
                logs.debug(`logging into Tumblr`);

                await downloader.login(username, password);

                logs.info(`login as ${username} success beginning download of blog "${blogSubdomain}"`);

                let skipExistingFilter = (photo) => {
                    let photoPath = getPhotoPath(dir, photo);
                    if (fs.existsSync(photoPath)) {
                        logs.debug(`photo "${photo.photoId}" has already been saved to "${photoPath}". skipping`);
                        return false;
                    }
                    return true;
                };

                await downloader.scrapeBlog({
                    blogSubdomain,
                    stopAtIndex,
                    stopAtPage,
                    downloadPhotos: true,
                    pageNumber,
                    predownloadFilter: (nconf.get('skipExisting') && skipExistingFilter)
                });

                logs.info(`download complete. exiting.`);

                return 0;
            } catch (error) {
                logs.error(`Error downloading from blog ${blogSubdomain}: ${error.message}`);
                return 1;
            }
        })();

        p.then(resolve).catch(reject);
    });
}

/**
 * Main function for the application.
 * @async
 * @returns {Promise} - Returns the exit code.
 */
async function main () {
    var command;

    const yargs = require('yargs')
    .version(pkg.version)
    .usage('Usage: tumblr-image-downloader [command] [arguments]')
    .strict()
    .option('logLevel', {
        alias: 'l',
        describe: 'Sets the verbosity of log output',
        default: 'info'
    })
    .option('quiet', {
        alias: 'q',
        describe: 'Turns logging off',
        default: false
    })
    .option('username', {
        alias: 'u',
        describe: 'Username to login with'
    })
    .option('password', {
        alias: 'p',
        describe: 'Password to login with'
    })
    .option('config', {
        alias: 'f',
        describe: 'A JSON configuration file to read from'
    })
    .option('proxyUrl', {
        alias: 'x',
        describe: 'The url to a proxy that will be used for all requests. SOCKS(4/5), HTTP(S) and PAC accepted.'
    })
    .command([ '$0', 'download [blogSubdomain] [directory]' ], 'Downloads all photos in a Tumblr blog', (yargs) => {
        yargs
            .positional('blogSubdomain', {
                describe: 'Blog to download',
                demand: true
            })
            .positional('directory', {
                describe: 'Directory to download to',
                demand: true
            })
            .option('pageNumber', {
                alias: 'n',
                describe: 'Page number to start downloading from'
            })
            .option('stopAtIndex', {
                alias: 'i',
                describe: 'Stops downloading after this many pages'
            })
            .option('stopAtPageNumber', {
                alias: 'r',
                describe: "Stop downloading when this page number has been reached"
            })
            .option('skipExisting', {
                alias: 'e',
                describe: 'Skip downloading existing photos.',
                default: true
            });
    }, (argv) => { 
        let args = argv._;
        if (args.length > 2)
            args.shift();
        command = download.bind(null, args); 
    })

    nconf = new Provider();

    nconf
        .argv(yargs)
        .env({
            whitelist: env_whitelist.concat(env_whitelist.map(env_to_config)),
            parseValues: true,
            separator: '__',
            transform: (obj) => {
                if (env_whitelist.includes(obj.key)) {
                    if (obj.key.indexOf('_') !== -1) {
                        obj.key = env_to_config(obj.key.toLowerCase().replace('tumblr_', ''));
                    }
                }
                return obj;
            }
        })
        .defaults(require('./default_config'));

    logs = winston.createLogger({
        level: nconf.get('logLevel'),
        format: winston.format.simple(),
        silent: nconf.get('quiet'),
        transports: [
            new winston.transports.Console({ silent: nconf.get('quiet') })
        ]
    });

    if (nconf.get('config'))
        nconf.file({ file: nconf.get('config') });

    let code = await command(nconf, logs);
    if (code)
        process.exit(code);
}

/**
 * This module contains the command-line logic for the application.
 * @module tumblr-image-downloader/launch
 */
module.exports = { 
    main, 
    download
};