index.ts

import puppeteer, { Page } from "puppeteer";
import { PDFDocument } from "pdf-lib"
import * as readline from 'readline';
import imageSize from 'image-size';
import fs from "fs";

import { dirname } from 'path';
import { fileURLToPath } from 'url';

// import cred from './dev.js';    //DEV

(async () => {
    console.log('Welcome to UniTO-book-scrape! This is a CLI utility for downloading books from the UniTO library 📚');
    console.log('Access to this application and the generated PDF documents is reserved for enrolled UniTO students with valid credentials 🔐');
    console.log('Do NOT share any PDF generated by this application ❌ The developer takes no responsibility for any improper use of the application 🙉');

    console.log('\nThis application is open source, view the code at https://github.com/ornato-t/UniTO-book-scrape 💻');
    console.log('Your UniTO credentials are safe! This application doesn\'t save them and only uses them for the authenthication process 👀\n')

    const { USERNAME, PWD, BOOK } = await getCredentials();
    // const { USERNAME, PWD, BOOK } = cred;   //DEV

    if (USERNAME === '' || PWD === '' || BOOK === '') {
        console.log('Error, inputs can\'t be empty');
        process.exit();
    }

    const { code, type } = checkURL(BOOK);

    const browser = await puppeteer.launch();
    const page = await browser.newPage();

    console.log('\nAttempting login');
    await login(page, USERNAME, PWD);
    console.log('Login succesful\n');

    const title = await getBookTitle(page, code, type);

    //Create new downloads dir
    if (!fs.existsSync('downloads')) {
        fs.mkdirSync('downloads');
    }
    //Create new html dir
    if (!fs.existsSync('html')) {
        fs.mkdirSync('html');
    }

    let pageNum = 0;
    let path: string | null;   //Last HTML file generated
    const pathList: string[] = []; //Array of paths to HTML files

    console.log(`\nBeginning download of: "${title}"`)
    do {
        pageNum++;
        console.log('Downloading page ', pageNum)
        const sourcePaths = await downloadPage(page, code, type, pageNum);
        path = generateHTML(sourcePaths, pageNum);
        if (path != null) pathList.push(path);
    } while (path !== null)

    console.log(`\nDownload finished at page ${pageNum}\n`);

    await browser.close();

    const { width, height } = getImageSize(pathList)

    console.log('Assembling into a PDF file\n');
    await generatePdf(title, pathList, width, height);
    console.log('\nDone, cleaning up...\n');

    fs.rm('downloads', { recursive: true }, err => { if (err) throw err; });
    fs.rm('html', { recursive: true }, err => { if (err) throw err; });

    console.log('Execution complete, exiting\n');
})();

//Check a URL and assign a type to it. Either "text" or "num"
function checkURL(url: string) {
    /*
    Match a UniTO book URL. The "s" in "https" and the page number are optional
    textRegex matches the url of a book code containing text
    numRegex matches the url of a numeric book code
    */
    const textRegex = /https?:\/\/unito\.studenti33\.it\/secure\/docs\/([\w-]+)(?:\/[0-9]+)?\/index\.html/;
    const numRegex = /https?:\/\/unito\.studenti33\.it\/secure\/docs\/([0-9]+)\/HTML(?:\/[0-9]+)?\/index\.html/;

    let type: 'text' | 'num';
    let code: string;

    //Extract book code via regex capture group
    if (numRegex.test(url)) {
        code = url.match(numRegex)?.[1] ?? '';
        type = 'num';
        
    } else if (textRegex.test(url)) {
        code = url.match(textRegex)?.[1] ?? '';
        type = 'text';

    } else {
        console.log('Error, invalid book URL');
        process.exit();
    }

    return { code, type };
}

async function getCredentials(): Promise<{ USERNAME: string, PWD: string, BOOK: string }> {
    const rl = readline.createInterface({
        input: process.stdin,
        output: process.stdout,
    });

    const USERNAME = await prompt('Inserisci il tuo nome utente UniTO / Enter your UniTO username ');
    const PWD = await prompt('Inserisci la tua password UniTO / Enter your UniTO password ');
    const BOOK = await prompt('Inserisci l\'URL (link) del libro desiderato / Enter the URL (link) of the desired book ');
    rl.close();

    return { USERNAME, PWD, BOOK };

    function prompt(question: string): Promise<string> {
        return new Promise((resolve) => {
            rl.question(question, (answer) => {
                resolve(answer);
            });
        });
    }
}

//Login to the UniTO intranet
async function login(page: Page, username: string, password: string) {
    const URL = 'https://my.unito.it'

    //Move to login page and wait for it to load
    await page.goto(URL, { waitUntil: 'networkidle0' });

    //Type username and password
    await page.type('input[name="j_username"]', username);
    await page.type('input[name="j_password"]', password);

    //Click button
    await Promise.all([page.waitForNavigation(), page.click('button[name="_eventId_proceed"]')]);

    //Wait for series of redirects to be over - we want to be sure that we're logged in
    await page.waitForNetworkIdle();
}

//Loads the book's first page and returns its title
async function getBookTitle(page: Page, code: string, type: "num" | "text") {
    let url: string;
    if (type === 'num') {
        url = `http://unito.studenti33.it/secure/docs/${code}/HTML/1/index.html`;
    } else if (type === 'text') {
        url = `http://unito.studenti33.it/secure/docs/${code}/1/index.html`;
    } else {
        return 'Unknown';
    }

    await page.goto(url, { waitUntil: 'networkidle0' });

    return await page.title();
}

//Download a page's text and background
async function downloadPage(page: Page, bookCode: string, type: 'num' | 'text', pageNum: number): Promise<HTMLPage> {
    const imgUrl = (() => {
        if (type === 'num') return `https://unito.studenti33.it/secure/docs/${bookCode}/HTML//files/assets/common/page-html5-substrates/page${pageNumFixed(pageNum)}_1.jpg?uni=557d76170c245168845e5673708d98fd`;
        else if (type === 'text') return `http://unito.studenti33.it/secure/docs/${bookCode}/files/assets/common/page-html5-substrates/page${pageNumFixed(pageNum)}_2.jpg?uni=02c8998da95b857ae7ea9a28539cd252`;
        else return '';
    })();
    const textUrl = (() => {
        if (type === 'num') return `http://unito.studenti33.it/secure/docs/${bookCode}/HTML//files/assets/common/page-vectorlayers/${pageNumFixed(pageNum)}.svg?uni=557d76170c245168845e5673708d98fd`;
        else if (type === 'text') return `http://unito.studenti33.it/secure/docs/${bookCode}/files/assets/common/page-vectorlayers/${pageNumFixed(pageNum)}.svg?uni=02c8998da95b857ae7ea9a28539cd252`;
        else return '';
    })();

    const outPath = `downloads/${pageNum}`;
    let textCode = 200, imgCode = 200;
    const paths: HTMLPage = new Object();

    //Fetch text page and download it, return in case of error
    try {
        const res = await page.goto(textUrl, { waitUntil: 'networkidle0' });
        if (res != null)
            if (res.status() >= 400) {
                textCode = res.status();
                console.log(`\tError ${textCode} while downloading ${pageNum}.svg`);
            } else {
                fs.writeFileSync(outPath + '.svg', await page.content());
                paths.fontFile = outPath + '.svg';
            }
    } catch (e) { console.log(e); }

    //Fetch image page and download it, return in case of error
    try {
        const res = await page.goto(imgUrl, { waitUntil: 'load' });
        if (res != null)
            if (res.status() >= 400) {
                imgCode = res.status();
                console.log(`\tError ${imgCode} while downloading ${pageNum}.jpg`);
            } else {
                fs.writeFileSync(outPath + '.jpeg', new Uint8Array(await res.buffer()));
                paths.backgroundImage = outPath + '.jpeg';
            }
    } catch (e) { console.log(e); }

    return paths;
    //Return a string containing a number with leading zeros, always 4 characters long
    function pageNumFixed(n: number) {
        switch (n.toString().length) {
            case 1:
                return `000${n}`;
            case 2:
                return `00${n}`;
            case 3:
                return `0${n}`;
            case 4:
                return `${n}`;
        }
    }
}

//Combine a .jpeg image and some .svg text to form a .html page. Return the path to that page or null in case of an error
function generateHTML(paths: HTMLPage, page: number) {
    const outPath = `./html/${page}.html`;
    if (paths.backgroundImage !== undefined && paths.fontFile !== undefined) {  //Both background and text
        const html = `<!DOCTYPE html> <html> <head>
        <title>Pagina ${page}</title> 
        <style> html, body { height: 100%; margin: 0; padding: 0; } .background { position: relative; height: 100%; width: 100%; } .background img,object { position: absolute; top: 0; left: 0; width: 100%; height: 100%; object-fit: cover; } </style> </head> <body> <div class="background">
        <img draggable="false" src="../${paths.backgroundImage}">
        <object type="image/svg+xml" data="../${paths.fontFile}"></object>
        </div> </body> </html>`;

        fs.writeFileSync(outPath, html);
        return outPath;
    } else if (paths.backgroundImage !== undefined) {   //Only background, no text
        const html = `<!DOCTYPE html> <html> <head>
        <title>Pagina ${page}</title> 
        <style> html, body { height: 100%; margin: 0; padding: 0; } .background { position: relative; height: 100%; width: 100%; } .background img,object { position: absolute; top: 0; left: 0; width: 100%; height: 100%; object-fit: cover; } </style> </head> <body> <div class="background">
        <img draggable="false" src="../${paths.backgroundImage}">
        </div> </body> </html>`;

        fs.writeFileSync(outPath, html);
        return outPath;
    } else if (paths.fontFile !== undefined) {  //Only text no background (unlikely)
        const html = `<!DOCTYPE html> <html> <head>
        <title>Pagina ${page}</title> 
        <style> html, body { height: 100%; margin: 0; padding: 0; } .background { position: relative; height: 100%; width: 100%; } .background img,object { position: absolute; top: 0; left: 0; width: 100%; height: 100%; object-fit: cover; } </style> </head> <body> <div class="background">
        <object type="image/svg+xml" data="../${paths.fontFile}"></object>
        </div> </body> </html>`;

        fs.writeFileSync(outPath, html);
        return outPath;
    }

    return null;
}

//Returns the width and height of the last image of the path list
function getImageSize(paths: string[]) {
    const regex = /\.\/html\/([0-9]+)\.html/;   //Regex to extract the page number

    //If no last page found, decrease by one and retry
    for (let i = paths.length - 1; i >= 0; i--) {
        const html = paths[i] ?? './html/1.html';    //Path of the last image in the book
        const pageCode = html.match(regex)?.[1] ?? '1';  //Number of the html page
        const page = `./downloads/${pageCode}.jpeg`;
        try {
            const dims = imageSize(page);
            return { width: dims.width ?? 500, height: dims.height ?? 600 }
        } catch (_) {
            console.log(`No image found at page ${pageCode}, trying ${Number.parseInt(pageCode) - 1}`)
        }
    }

    return { width: 500, height: 600 }; //If everything else fails, return default value
}

async function generatePdf(outPath: string, paths: string[], width: number, height: number) {
    const pdfDoc = await PDFDocument.create();

    const browser = await puppeteer.launch({ headless: true });
    const page = await browser.newPage();
    await page.setViewport({ height: height, width: width });

    page.on('console', (msg) => console.log('PAGE LOG:', msg.text()));
    page.on('pageerror', (err) => console.log('PAGE ERROR:', err));

    let pageNum = 0;
    for (const path of paths) {
        pageNum++;
        console.log('Assembling page', pageNum)
        const pdfBytes = await generatePdfSinglePage(path, page, width, height);
        const pdfDocBytes = await PDFDocument.load(pdfBytes);
        const [pdfDocPage] = await pdfDoc.copyPages(pdfDocBytes, [0]);
        pdfDoc.addPage(pdfDocPage);
    }

    await browser.close();


    // Save the PDF to a file
    const pdfBytes = await pdfDoc.save();
    fs.writeFileSync(outPath + '.pdf', pdfBytes);


    async function generatePdfSinglePage(html: string, page: Page, width: number, height: number) {
        await page.goto(getPath(html), { waitUntil: 'networkidle0' });

        // Wait for the page to finish loading
        await page.waitForSelector('img');


        // Set the PDF dimensions
        const pdfOptions = {
            width: `${width}px`,
            height: `${height}px`,
            printBackground: true, // Capture background colors and images
        };

        // Generate the PDF
        const pdfBuffer = await page.pdf(pdfOptions);

        return pdfBuffer;

        function getPath(file: string) {
            //If run with "npm start" manually fetch dir name
            if (process.argv[2] === 'es') return dirname(fileURLToPath(import.meta.url)) + file.slice(1);

            //If run from commonJS file dirname is already defined
            return __dirname + file.slice(1);
        }
    }
}

interface HTMLPage {
    backgroundImage?: string,
    fontFile?: string
}