Skip to content

Commit

Permalink
Various package changes with cheerio (#114)
Browse files Browse the repository at this point in the history 
* package.json: Use latest cheerio package and pin type def version

* Fix McReader and update cheerio syntax
  • Loading branch information
@niclimcy
niclimcy authored Aug 16, 2024
1 parent 4da62be commit c65c79a
Show file tree
Hide file tree
Showing 26 changed files with 227 additions and 214 deletions.
17 changes: 8 additions & 9 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,19 @@
"author": "Netsky",
"license": "GPL-3.0-or-later",
"devDependencies": {
"@tsconfig/recommended": "^1.0.1",
"@types/cheerio": "^0.22.21",
"@typescript-eslint/eslint-plugin": "^5.1.0",
"@typescript-eslint/parser": "^5.57.1",
"eslint": "^7.27.0",
"@tsconfig/recommended": "^1.0.7",
"@typescript-eslint/eslint-plugin": "^5.62.0",
"@typescript-eslint/parser": "^5.62.0",
"eslint": "^7.32.0",
"eslint-plugin-modules-newline": "^0.0.6",
"typescript": "^4.9.5"
},
"dependencies": {
"@paperback/toolchain": "^0.8.0-alpha.47",
"@paperback/types": "^0.8.0-alpha.47",
"cheerio": "^1.0.0-rc.12",
"@paperback/toolchain": "^0.8.7",
"@paperback/types": "^0.8.7",
"cheerio": "^1.0.0",
"html-entities": "^2.5.2",
"nodemon": "^2.0.22",
"ts-node": "^10.2.1"
"ts-node": "^10.9.2"
}
}
19 changes: 9 additions & 10 deletions src/ComicOnlineFree/ComicOnlineFree.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import {
PagedResults,
SourceInfo,
ContentRating,
BadgeColor,
Request,
Response,
TagSection,
Expand All @@ -19,6 +18,8 @@ import {
HomePageSectionsProviding
} from '@paperback/types'

import * as cheerio from 'cheerio'

import {
parseChapterDetails,
isLastPage,
Expand Down Expand Up @@ -49,8 +50,6 @@ export const ComicOnlineFreeInfo: SourceInfo = {

export class ComicOnlineFree implements SearchResultsProviding, MangaProviding, ChapterProviding, HomePageSectionsProviding {

constructor(private cheerio: CheerioAPI) { }

requestManager = App.createRequestManager({
requestsPerSecond: 4,
requestTimeout: 15000,
Expand Down Expand Up @@ -81,7 +80,7 @@ export class ComicOnlineFree implements SearchResultsProviding, MangaProviding,

const response = await this.requestManager.schedule(request, 1)
this.CloudFlareError(response.status)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
return parseMangaDetails($, mangaId)
}

Expand All @@ -93,7 +92,7 @@ export class ComicOnlineFree implements SearchResultsProviding, MangaProviding,

const response = await this.requestManager.schedule(request, 1)
this.CloudFlareError(response.status)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
return parseChapters($, mangaId)
}

Expand All @@ -105,7 +104,7 @@ export class ComicOnlineFree implements SearchResultsProviding, MangaProviding,

const response = await this.requestManager.schedule(request, 1)
this.CloudFlareError(response.status)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
return parseChapterDetails($, mangaId, chapterId)
}

Expand All @@ -117,7 +116,7 @@ export class ComicOnlineFree implements SearchResultsProviding, MangaProviding,

const response = await this.requestManager.schedule(request, 1)
this.CloudFlareError(response.status)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
parseHomeSections($, sectionCallback)
}

Expand All @@ -142,7 +141,7 @@ export class ComicOnlineFree implements SearchResultsProviding, MangaProviding,

const response = await this.requestManager.schedule(request, 1)
this.CloudFlareError(response.status)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
const manga = parseViewMore($)

metadata = !isLastPage($) ? { page: page + 1 } : undefined
Expand All @@ -159,7 +158,7 @@ export class ComicOnlineFree implements SearchResultsProviding, MangaProviding,
})

const response = await this.requestManager.schedule(request, 1)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
return parseTags($)
}

Expand All @@ -178,7 +177,7 @@ export class ComicOnlineFree implements SearchResultsProviding, MangaProviding,
})

const response = await this.requestManager.schedule(request, 1)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
const manga = parseSearch($)
metadata = !isLastPage($) ? { page: page + 1 } : undefined
return App.createPagedResults({
Expand Down
17 changes: 9 additions & 8 deletions src/ComicOnlineFree/ComicOnlineFreeParser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ import {
} from '@paperback/types'

import { decode as decodeHTMLEntity } from 'html-entities'
import { CheerioAPI } from 'cheerio'

export const parseMangaDetails = ($: CheerioStatic, mangaId: string): SourceManga => {
export const parseMangaDetails = ($: CheerioAPI, mangaId: string): SourceManga => {
const titles: string[] = []

titles.push(decodeHTMLEntity($('td:contains(Name:)').first().next().text().trim()))
Expand Down Expand Up @@ -61,7 +62,7 @@ export const parseMangaDetails = ($: CheerioStatic, mangaId: string): SourceMang
})
}

export const parseChapters = ($: CheerioStatic, mangaId: string): Chapter[] => {
export const parseChapters = ($: CheerioAPI, mangaId: string): Chapter[] => {
const chapters: Chapter[] = []
let sortingIndex = 0

Expand Down Expand Up @@ -101,7 +102,7 @@ export const parseChapters = ($: CheerioStatic, mangaId: string): Chapter[] => {
})
}

export const parseChapterDetails = ($: CheerioStatic, mangaId: string, chapterId: string): ChapterDetails => {
export const parseChapterDetails = ($: CheerioAPI, mangaId: string, chapterId: string): ChapterDetails => {
const pages: string[] = []

for (const img of $('img', 'div.chapter-container').toArray()) {
Expand All @@ -119,7 +120,7 @@ export const parseChapterDetails = ($: CheerioStatic, mangaId: string, chapterId
return chapterDetails
}

export const parseHomeSections = ($: CheerioStatic, sectionCallback: (section: HomeSection) => void): void => {
export const parseHomeSections = ($: CheerioAPI, sectionCallback: (section: HomeSection) => void): void => {
const popularSection = App.createHomeSection({
id: 'popular',
title: 'Popular Comics',
Expand Down Expand Up @@ -182,7 +183,7 @@ export const parseHomeSections = ($: CheerioStatic, sectionCallback: (section: H
sectionCallback(updateSection)
}

export const parseViewMore = ($: CheerioStatic): PartialSourceManga[] => {
export const parseViewMore = ($: CheerioAPI): PartialSourceManga[] => {
const manga: PartialSourceManga[] = []
const collectedIds: string[] = []

Expand All @@ -207,7 +208,7 @@ export const parseViewMore = ($: CheerioStatic): PartialSourceManga[] => {
return manga
}

export const parseTags = ($: CheerioStatic): TagSection[] => {
export const parseTags = ($: CheerioAPI): TagSection[] => {
const arrayTags: Tag[] = []

for (const tag of $('li', 'ul.search-checks').toArray()) {
Expand All @@ -222,7 +223,7 @@ export const parseTags = ($: CheerioStatic): TagSection[] => {
return tagSections
}

export const parseSearch = ($: CheerioStatic): PartialSourceManga[] => {
export const parseSearch = ($: CheerioAPI): PartialSourceManga[] => {
const mangas: PartialSourceManga[] = []

for (const obj of $('div.manga-box', 'div.result-left').toArray()) {
Expand All @@ -243,7 +244,7 @@ export const parseSearch = ($: CheerioStatic): PartialSourceManga[] => {
return mangas
}

export const isLastPage = ($: CheerioStatic): boolean => {
export const isLastPage = ($: CheerioAPI): boolean => {
let isLast = false
const items: string[] = []

Expand Down
8 changes: 4 additions & 4 deletions src/DynastyScans/DynastyScans.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import {
PartialSourceManga
} from '@paperback/types'

import * as cheerio from 'cheerio'

import {
Chapters,
Series,
Expand All @@ -43,8 +45,6 @@ export const DynastyScansInfo: SourceInfo = {

export class DynastyScans implements SearchResultsProviding, MangaProviding, ChapterProviding, HomePageSectionsProviding {

constructor(private cheerio: CheerioAPI) { }

requestManager = App.createRequestManager({
requestsPerSecond: 10,
requestTimeout: 20000,
Expand Down Expand Up @@ -92,7 +92,7 @@ export class DynastyScans implements SearchResultsProviding, MangaProviding, Cha

let description
if (data.description) {
const $ = this.cheerio.load(`<div>${data.description}</div>`)
const $ = cheerio.load(`<div>${data.description}</div>`)
description = $('div').text().trim()
} else {
description = `Tags: ${data.tags.map((x: { name: string }) => x.name).join(', ')}`
Expand Down Expand Up @@ -459,7 +459,7 @@ export class DynastyScans implements SearchResultsProviding, MangaProviding, Cha
})

const response = await this.requestManager.schedule(request, 1)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)

const results: PartialSourceManga[] = []

Expand Down
16 changes: 8 additions & 8 deletions src/HentaiCosplay/HentaiCosplay.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import {
Tag
} from '@paperback/types'

import * as cheerio from 'cheerio'

import {
parseChapterDetails,
isLastPage,
Expand Down Expand Up @@ -51,8 +53,6 @@ export const HentaiCosplayInfo: SourceInfo = {

export class HentaiCosplay implements SearchResultsProviding, MangaProviding, ChapterProviding, HomePageSectionsProviding {

constructor(private cheerio: CheerioAPI) { }

requestManager = App.createRequestManager({
requestsPerSecond: 4,
requestTimeout: 15000,
Expand Down Expand Up @@ -83,7 +83,7 @@ export class HentaiCosplay implements SearchResultsProviding, MangaProviding, Ch

const response = await this.requestManager.schedule(request, 1)
this.CloudFlareError(response.status)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
return parseMangaDetails($, mangaId)
}

Expand All @@ -100,7 +100,7 @@ export class HentaiCosplay implements SearchResultsProviding, MangaProviding, Ch

const response = await this.requestManager.schedule(request, 1)
this.CloudFlareError(response.status)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
return parseChapterDetails($, mangaId)
}

Expand All @@ -112,7 +112,7 @@ export class HentaiCosplay implements SearchResultsProviding, MangaProviding, Ch

const response = await this.requestManager.schedule(request, 1)
this.CloudFlareError(response.status)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
parseHomeSections($, sectionCallback)
}

Expand Down Expand Up @@ -143,7 +143,7 @@ export class HentaiCosplay implements SearchResultsProviding, MangaProviding, Ch

const response = await this.requestManager.schedule(request, 1)
this.CloudFlareError(response.status)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
const manga = parseViewMore($)

metadata = !isLastPage($) ? { page: page + 1 } : undefined
Expand All @@ -160,7 +160,7 @@ export class HentaiCosplay implements SearchResultsProviding, MangaProviding, Ch
})

const response = await this.requestManager.schedule(request, 1)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
return parseTags($)
}

Expand All @@ -181,7 +181,7 @@ export class HentaiCosplay implements SearchResultsProviding, MangaProviding, Ch
}

const response = await this.requestManager.schedule(request, 1)
const $ = this.cheerio.load(response.data as string)
const $ = cheerio.load(response.data as string)
const manga = parseViewMore($)

metadata = !isLastPage($) ? { page: page + 1 } : undefined
Expand Down
13 changes: 7 additions & 6 deletions src/HentaiCosplay/HentaiCosplayParser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ import {
} from '@paperback/types'

import { decode as decodeHTMLEntity } from 'html-entities'
import { CheerioAPI } from 'cheerio'

export const parseMangaDetails = ($: CheerioStatic, mangaId: string): SourceManga => {
export const parseMangaDetails = ($: CheerioAPI, mangaId: string): SourceManga => {
const title: string = decodeHTMLEntity($('span#title > h2').text().trim())

const image = $('img', $('div.icon-overlay')).first().attr('src') ?? ''
Expand Down Expand Up @@ -47,7 +48,7 @@ export const parseChapters = (mangaId: string): Chapter[] => {
})]
}

export const parseChapterDetails = ($: CheerioStatic, mangaId: string): ChapterDetails => {
export const parseChapterDetails = ($: CheerioAPI, mangaId: string): ChapterDetails => {
const pages: string[] = []


Expand All @@ -65,7 +66,7 @@ export const parseChapterDetails = ($: CheerioStatic, mangaId: string): ChapterD
return chapterDetails
}

export const parseHomeSections = ($: CheerioStatic, sectionCallback: (section: HomeSection) => void): void => {
export const parseHomeSections = ($: CheerioAPI, sectionCallback: (section: HomeSection) => void): void => {
const sections = [
{
sectionID: App.createHomeSection({
Expand Down Expand Up @@ -122,7 +123,7 @@ export const parseHomeSections = ($: CheerioStatic, sectionCallback: (section: H
}
}

export const parseViewMore = ($: CheerioStatic): PartialSourceManga[] => {
export const parseViewMore = ($: CheerioAPI): PartialSourceManga[] => {
const manga: PartialSourceManga[] = []
const collectedIds: string[] = []

Expand All @@ -145,7 +146,7 @@ export const parseViewMore = ($: CheerioStatic): PartialSourceManga[] => {
return manga
}

export const parseTags = ($: CheerioStatic): TagSection[] => {
export const parseTags = ($: CheerioAPI): TagSection[] => {
const arrayTags: Tag[] = []
for (const tag of $('#tags li').toArray()) {
const id = $('a', tag).attr('href')?.trim().replace(/\/$/, '').split('/').pop()
Expand All @@ -157,7 +158,7 @@ export const parseTags = ($: CheerioStatic): TagSection[] => {
return tagSections
}

export const isLastPage = ($: CheerioStatic): boolean => {
export const isLastPage = ($: CheerioAPI): boolean => {
let isLast = false

const lastPage = $('div.wp-pagenavi > a.last').attr('href')
Expand Down
Loading

0 comments on commit c65c79a

Please sign in to comment.