From d6f54622262bb383b161a689a9f4d5b906231ff6 Mon Sep 17 00:00:00 2001 From: kmlbgn Date: Thu, 7 Mar 2024 20:46:35 +0800 Subject: [PATCH 1/2] fix: major refactoring --- src/plugins/internalLinks.ts | 184 +++++++++++----------- src/plugins/pluginTestRun.ts | 286 ----------------------------------- src/plugins/pluginTypes.ts | 4 +- src/pull.ts | 161 ++++++++------------ src/transform.ts | 11 +- 5 files changed, 162 insertions(+), 484 deletions(-) delete mode 100644 src/plugins/pluginTestRun.ts diff --git a/src/plugins/internalLinks.ts b/src/plugins/internalLinks.ts index 3f05feb..c520d4f 100644 --- a/src/plugins/internalLinks.ts +++ b/src/plugins/internalLinks.ts @@ -2,130 +2,124 @@ import { IDocuNotionContext, IPlugin } from "./pluginTypes"; import { warning, verbose } from "../log"; import { NotionPage } from "../NotionPage"; -// converts a url to a local link, if it is a link to a page in the Notion site -// only here for plugins, notion won't normally be giving us raw urls. -// If it finds a URL but can't find the page it points to, it will return undefined. -// If it doesn't find a match at all, it returns undefined. -export function convertInternalUrl( - context: IDocuNotionContext, - url: string -): string | undefined { +/** + * Converts an external URL to a local link if it's a link to a page on the Notion site. + * Intended for plugin use; normally, Notion doesn't provide raw URLs. + * Returns undefined if the page cannot be found or if no match is found. + */ +export function convertInternalUrl(context: IDocuNotionContext, url: string): string | undefined { const kGetIDFromNotionURL = /https:\/\/www\.notion\.so\/([a-z0-9]+).*/; const match = kGetIDFromNotionURL.exec(url); if (match === null) { - warning( - `[standardInternalLinkConversion] Could not parse link ${url} as a Notion URL` - ); + warning(`Could not parse link ${url} as a Notion URL`); return undefined; } + const id = match[1]; - const allTabsPages = context.allTabsPages; - // find the page where pageId matches hrefFromNotion - const targetPage = allTabsPages.find(p => { - return p.matchesLinkId(id); - }); + let targetPage: { page: NotionPage | undefined, tab: string | undefined } = { page: undefined, tab: undefined }; for (const tab in context.allTabsPages) { + const foundPage = context.allTabsPages[tab].find(p => p.matchesLinkId(id)); + if (foundPage) { + targetPage = { page: foundPage, tab: tab }; + break; // Exit the loop once a matching page is found + } + } - if (!targetPage) { - // About this situation. See https://github.com/sillsdev/docu-notion/issues/9 - warning( - `[standardInternalLinkConversion] Could not find the target of this link. Note that links to outline sections are not supported. ${url}. https://github.com/sillsdev/docu-notion/issues/9` - ); - return undefined; + if (targetPage.page && targetPage.tab) { + return convertLinkHref(context, targetPage.tab, targetPage.page, url); + } else { + warning(`Could not find the target of this link. Links to outline sections are not supported. ${url}.`); + return `${id}[broken link]`; } - // warning( - // `[standardInternalLinkConversion] Found the target for ${id}, passing ${url}` - // ); - return convertLinkHref(context, targetPage, url); } -// handles the whole markdown link, including the label -function convertInternalLink( - context: IDocuNotionContext, - markdownLink: string -): string { +/** + * Converts a markdown link to a local link if it's a Notion page link. + * Skips conversion for image links and links that cannot be resolved to local targets. + * Returns the original markdown link if it cannot be parsed or converted. + */ +function convertInternalLink(context: IDocuNotionContext, markdownLink: string): string { const linkRegExp = /\[([^\]]+)?\]\((?!mailto:)(https:\/\/www\.notion\.so\/[^)]+|\/[^),]+)\)/g; const match = linkRegExp.exec(markdownLink); if (match === null) { - warning( - `[InternalLinkPlugin] Could not parse link ${markdownLink}` - ); + warning(`Could not parse link ${markdownLink}`); return markdownLink; } - const labelFromNotion = match[1] || ""; let hrefFromNotion = match[2]; + const labelFromNotion = match[1] || ""; + const imageFileExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.svg']; + const isImageLink = imageFileExtensions.some(ext => hrefFromNotion.endsWith(ext)); + if (isImageLink) { + verbose(`${hrefFromNotion} is an internal image link and will be skipped.`); + return markdownLink; + } - - // TODO: This is a hotfix to dodge internal image links parsing - const imageFileExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.svg']; - const isImageLink = imageFileExtensions.some(ext => hrefFromNotion.endsWith(ext)); - if (isImageLink) { - verbose(`Link parsing: [InternalLinkPlugin] ${hrefFromNotion} is an internal image link and will be skipped. Make sure it exists !`); - return markdownLink; - } - - // Find the last occurrence of either '-' or '/' and take everything to the right to extract id and fragment const lastSpecialCharIndex = Math.max(hrefFromNotion.lastIndexOf('-'), hrefFromNotion.lastIndexOf('/')); if (lastSpecialCharIndex !== -1) { hrefFromNotion = hrefFromNotion.substring(lastSpecialCharIndex + 1); } - const allTabsPages = context.allTabsPages; - // find the page where pageId matches hrefFromNotion - const targetPage = allTabsPages.find(p => { - return p.matchesLinkId(hrefFromNotion); - }); - - if (!targetPage) { - // About this situation. See https://github.com/sillsdev/docu-notion/issues/9 - warning( - `Link parsing: [InternalLinkPlugin] Could not find a local target for ${hrefFromNotion}. Note that links to other notions pages or outline sections are not supported > https://github.com/sillsdev/docu-notion/issues/9` - ); - return `${labelFromNotion}†`; + let targetPage: { page: NotionPage | undefined, tab: string | undefined } = { page: undefined, tab: undefined }; for (const tab in context.allTabsPages) { + const foundPage = context.allTabsPages[tab].find(p => p.matchesLinkId(hrefFromNotion)); + if (foundPage) { + targetPage = { page: foundPage, tab: tab }; + break; + } } - const label = convertLinkLabel(targetPage, labelFromNotion); - const url = convertLinkHref(context, targetPage, hrefFromNotion); - return `[${label}](${url})`; + if (targetPage.page && targetPage.tab) { + const label = convertLinkLabel(targetPage.page, labelFromNotion); + const url = convertLinkHref(context, targetPage.tab, targetPage.page, hrefFromNotion); + return `[${label}](${url})`; + } else { + warning(`Could not find a local target for ${hrefFromNotion}.`); + return `${labelFromNotion}[broken link]`; + } } +/** + * Fixes the link label if it's a "mention" to display the page name or title instead. + */ function convertLinkLabel(targetPage: NotionPage, text: string): string { - // In Notion, if you just add a link to a page without linking it to any text, then in Notion - // you see the name of the page as the text of the link. But when Notion gives us that same - // link, it uses "mention" as the text. So we have to look up the name of the page in - // order to fix that.; - if (text !== "mention") return text; - else return targetPage.nameOrTitle; + return text === "mention" ? targetPage.nameOrTitle : text; } + +/** +* Converts the URL to a local link format based on the context of the current tab. +* Appends fragment identifiers to the link if they exist. +* Note: The official Notion API does not include links to headings unless they are part of an inline link. +*/ function convertLinkHref( - context: IDocuNotionContext, - targetPage: NotionPage, - url: string + context: IDocuNotionContext, + tab: string, + page: NotionPage, + url: string ): string { - let convertedLink = context.layoutStrategy.getLinkPathForPage(targetPage); - - /***************************** - NOTE: as of this writing, the official Notion API completely drops links - to headings, unless they are part of a inline link. - *******************************/ + let convertedLink: string; + // If the link is from the current tab, use the direct link path to the page + if (tab == context.currentTab) { + convertedLink = context.layoutStrategy.getLinkPathForPage(page); + } else { + // If the link is from a different tab, prepend '/' and the tab name to the link path + convertedLink = "/" + tab + context.layoutStrategy.getLinkPathForPage(page); + } - // Include the fragment (# and after) if it exists - const { fragmentId } = parseLinkId(url); - // Log only if fragmentId is not an empty string - if (fragmentId !== "") { - verbose(`Link parsing: [InternalLinkPlugin] Parsed ${url} and got Fragment ID: ${fragmentId}`); - } - convertedLink += fragmentId; + // Extract the fragment identifier from the URL, if it exists + const { fragmentId } = parseLinkId(url); + if (fragmentId !== "") { + verbose(`[InternalLinkPlugin] Extracted Fragment ID from ${url}: ${fragmentId}`); + } + convertedLink += fragmentId; - //verbose(`Converting Link ${url} --> ${convertedLink}`); - return convertedLink; + return convertedLink; } -// Parse the link ID to replace the base page ID (before the #) with its slug if exists, and replace the fragment (# and after) if exists. -export function parseLinkId(fullLinkId: string): { - baseLinkId: string; // before the # - fragmentId: string; // # and after -} { - const iHash: number = fullLinkId.indexOf("#"); + + +/** + * Extracts the base link ID and fragment identifier from a full link ID. + */ +export function parseLinkId(fullLinkId: string): { baseLinkId: string; fragmentId: string } { + const iHash = fullLinkId.indexOf("#"); if (iHash >= 0) { return { baseLinkId: fullLinkId.substring(0, iHash), @@ -135,14 +129,14 @@ export function parseLinkId(fullLinkId: string): { return { baseLinkId: fullLinkId, fragmentId: "" }; } +/** + * Plugin for converting internal links to local links within Notion pages. + * Handles both "raw" and "inline" link formats from the Notion or notion-md source. + */ export const standardInternalLinkConversion: IPlugin = { name: "InternalLinkPlugin", linkModifier: { - // from notion (or notion-md?) we get slightly different hrefs depending on whether the links is "inline" - // (has some other text that's been turned into a link) or "raw". - // Raw links come in without a leading slash, e.g. [mention](4a6de8c0-b90b-444b-8a7b-d534d6ec71a4) - // Inline links come in with a leading slash, e.g. [pointer to the introduction](/4a6de8c0b90b444b8a7bd534d6ec71a4) match: /\[([^\]]+)?\]\((?!mailto:)(https:\/\/www\.notion\.so\/[^)]+|\/[^),]+)\)/, convert: convertInternalLink, }, -}; \ No newline at end of file +}; diff --git a/src/plugins/pluginTestRun.ts b/src/plugins/pluginTestRun.ts deleted file mode 100644 index c1d9c28..0000000 --- a/src/plugins/pluginTestRun.ts +++ /dev/null @@ -1,286 +0,0 @@ -import { Client } from "@notionhq/client"; -import { GetPageResponse } from "@notionhq/client/build/src/api-endpoints"; -import { NotionToMarkdown } from "notion-to-md"; -import { IDocuNotionContext } from "./pluginTypes"; -import { HierarchicalNamedLayoutStrategy } from "../HierarchicalNamedLayoutStrategy"; -import { NotionPage } from "../NotionPage"; -import { getMarkdownFromNotionBlocks } from "../transform"; -import { IDocuNotionConfig } from "../config/configuration"; -import { NotionBlock } from "../types"; -import { convertInternalUrl } from "./internalLinks"; -import { numberChildrenIfNumberedList } from "../pull"; - -export async function blocksToMarkdown( - config: IDocuNotionConfig, - blocks: NotionBlock[], - tabsPages?: NotionPage[], - allTabsPages?: NotionPage[], - // Notes on children: - // - These children will apply to each block in blocks. (could enhance but not needed yet) - // - If you are passing in children, it is probably because your parent block has has_children=true. - // In that case, notion-to-md will make an API call... you'll need to set any validApiKey. - children?: NotionBlock[], - validApiKey?: string -): Promise { - const notionClient = new Client({ - auth: validApiKey || "unused", - }); - const notionToMD = new NotionToMarkdown({ - notionClient, - }); - - const docunotionContext: IDocuNotionContext = { - config: config, - notionToMarkdown: notionToMD, - getBlockChildren: (id: string) => { - // We call numberChildrenIfNumberedList here because the real getBlockChildren does - if (children) numberChildrenIfNumberedList(children); - - return new Promise((resolve, reject) => { - resolve(children ?? []); - }); - }, - convertNotionLinkToLocalDocusaurusLink: (url: string) => { - return convertInternalUrl(docunotionContext, url); - }, - imports: [], - - //TODO might be needed for some tests, e.g. the image transformer... - directoryContainingMarkdown: "not yet", - relativeFilePathToFolderContainingPage: "not yet", - layoutStrategy: new HierarchicalNamedLayoutStrategy(), - options: { - notionToken: "", - rootPage: "", - locales: [], - markdownOutputPath: "", - imgOutputPath: "", - imgPrefixInMarkdown: "", - statusTag: "", - }, - tabsPages: tabsPages ?? [], - allTabsPages: allTabsPages ?? [], - counts: { - output_normally: 0, - skipped_because_empty: 0, - skipped_because_status: 0, - }, - // enhance: this needs more thinking, how we want to do logging in tests - // one thing is to avoid a situation where we break people's tests that - // have come to rely on logs that we later tweak in some way. - // log: { - // error: (s: string) => { - // error(s); - // }, - // warning: (s: string) => { - // warning(s); - // }, - // info: (s: string) => { - // // info(s); - // }, - // verbose: (s: string) => { - // // verbose(s); - // }, - // debug: (s: string) => { - // // logDebug("Testrun-TODO", s); - // }, - // }, - }; - - if (tabsPages && tabsPages.length) { - console.log(tabsPages[0].matchesLinkId); - console.log(docunotionContext.tabsPages[0].matchesLinkId); - } - if (allTabsPages && allTabsPages.length) { - console.log(allTabsPages[0].matchesLinkId); - console.log(docunotionContext.tabsPages[0].matchesLinkId); - } - const r = await getMarkdownFromNotionBlocks( - docunotionContext, - config, - blocks - ); - //console.log("blocksToMarkdown", r); - return r; -} - -// This is used for things like testing links to other pages and frontmatter creation, -// when just testing what happens to individual blocks is not enough. -// after getting this, you can make changes to it, then pass it to blocksToMarkdown -export function makeSamplePageObject(options: { - slug?: string; - name?: string; - id?: string; -}): NotionPage { - let slugObject: any = { - Slug: { - id: "%7D%3D~K", - type: "rich_text", - rich_text: [], - }, - }; - - if (options.slug) - slugObject = { - id: "%7D%3D~K", - type: "rich_text", - rich_text: [ - { - type: "text", - text: { - content: options.slug, - link: null, - }, - annotations: { - bold: false, - italic: false, - strikethrough: false, - underline: false, - code: false, - color: "default", - }, - plain_text: options.slug, - href: null, - }, - ], - }; - - const id = options.id || "4a6de8c0-b90b-444b-8a7b-d534d6ec71a4"; - const m: GetPageResponse = { - object: "page", - id: id, - created_time: "2022-08-08T21:07:00.000Z", - last_edited_time: "2023-01-03T14:38:00.000Z", - created_by: { - object: "user", - id: "11fb7f16-0560-4aee-ab88-ed75a850cfc4", - }, - last_edited_by: { - object: "user", - id: "11fb7f16-0560-4aee-ab88-ed75a850cfc4", - }, - cover: null, - icon: null, - parent: { - type: "database_id", - database_id: "c13f520c-06ad-41e4-a021-bdc2841ab24a", - }, - archived: false, - properties: { - Keywords: { - id: "%3F%7DLZ", - type: "rich_text", - rich_text: [], - }, - Property: { - id: "GmKE", - type: "rich_text", - rich_text: [], - }, - Label: { - id: "Phor", - type: "multi_select", - multi_select: [], - }, - Status: { - id: "oB~%3D", - type: "select", - select: { - id: "1", - name: "Ready For Review", - color: "red", - }, - }, - Authors: { - id: "tA%3BF", - type: "multi_select", - multi_select: [], - }, - Slug: slugObject, - Name: { - id: "title", - type: "title", - title: [ - { - type: "text", - text: { - content: options.name || "Hello World", - link: null, - }, - annotations: { - bold: false, - italic: false, - strikethrough: false, - underline: false, - code: false, - color: "default", - }, - plain_text: options.name || "Hello World", - href: null, - }, - ], - }, - }, - url: `https://www.notion.so/Hello-World-${id}`, - }; - - const p = new NotionPage({ - layoutContext: "/Second-Level/Third-Level", - parentId: "d20d8391-b365-42cb-8821-cf3c5382c6ed", - pageId: id, - order: 0, - metadata: m, - foundDirectlyInOutline: false, - }); - - console.log(p.matchesLinkId); - - return p; -} - -export async function oneBlockToMarkdown( - config: IDocuNotionConfig, - block: object, - targetPage?: NotionPage -): Promise { - // just in case someone expects these other properties that aren't normally relevant, - // we merge the given block properties into an actual, full block - const fullBlock = { - ...{ - object: "block", - id: "937e77e5-f058-4316-9805-a538e7b4082d", - parent: { - type: "page_id", - page_id: "d20d8391-b365-42cb-8821-cf3c5382c6ed", - }, - created_time: "2023-01-13T16:33:00.000Z", - last_edited_time: "2023-01-13T16:33:00.000Z", - created_by: { - object: "user", - id: "11fb7f16-0560-4aee-ab88-ed75a850cfc4", - }, - last_edited_by: { - object: "user", - id: "11fb7f16-0560-4aee-ab88-ed75a850cfc4", - }, - has_children: false, - archived: false, - }, - ...block, - }; - - const dummyPage1 = makeSamplePageObject({ - slug: "dummy1", - name: "Dummy1", - }); - const dummyPage2 = makeSamplePageObject({ - slug: "dummy2", - name: "Dummy2", - }); - return await blocksToMarkdown( - config, - [fullBlock as NotionBlock], - targetPage ? [dummyPage1, targetPage] : undefined, - targetPage ? [dummyPage1, targetPage, dummyPage2] : undefined, - ); -} diff --git a/src/plugins/pluginTypes.ts b/src/plugins/pluginTypes.ts index dd31a73..0178108 100644 --- a/src/plugins/pluginTypes.ts +++ b/src/plugins/pluginTypes.ts @@ -74,8 +74,8 @@ export type IDocuNotionContext = { notionToMarkdown: NotionToMarkdown; directoryContainingMarkdown: string; relativeFilePathToFolderContainingPage: string; - tabsPages: NotionPage[]; - allTabsPages: NotionPage[]; + allTabsPages: Record; + currentTab: string; convertNotionLinkToLocalDocusaurusLink: (url: string) => string | undefined; counts: ICounts; diff --git a/src/pull.ts b/src/pull.ts index 6a48fa8..c3ff43f 100644 --- a/src/pull.ts +++ b/src/pull.ts @@ -42,8 +42,8 @@ export type DocuNotionOptions = { let layoutStrategy: LayoutStrategy; let notionToMarkdown: NotionToMarkdown; -let allTabsPages= new Array(); -let tabsPages: Array; +let allTabsPages: Record = {}; +let currentTabPages: Array; let counts = { output_normally: 0, skipped_because_empty: 0, @@ -90,7 +90,18 @@ export async function notionPull(options: DocuNotionOptions): Promise { // Create a base folder using markdownOutputPath (default "tabs") await fs.mkdir(options.markdownOutputPath.replace(/\/+$/, ""), { recursive: true }); + + //TODO group stage 1 should be extracted from getTabs to here await getTabs(options, "", "root", options.rootPage); + + group( + `Stage 2: Convert ${currentTabPages.length} Notion pages to markdown and save locally...` + ); + // Load config + const config = await loadConfigAsync(); + + await outputPages(options, config, allTabsPages); + endGroup(); } async function getTabs( @@ -99,7 +110,7 @@ async function getTabs( parentId: string, pageId: string, ) { - // Get root page metadata + // Create root page to fetch metadata const rootPage = await fromPageId( "", parentId, @@ -109,10 +120,7 @@ async function getTabs( false ); - // Load config - const config = await loadConfigAsync(); - - // Get tabs list + // Get all tabs (sub-pages of root page) const r = await getBlockChildren(rootPage.pageId); const pageInfo = await rootPage.getContentInfo(r); @@ -121,7 +129,7 @@ async function getTabs( // Recursively process each tabs for (const tabPageInfo of pageInfo.childPageIdsAndOrder) { // Get tabs page metadata - const tabs = await fromPageId( + const currentTab = await fromPageId( incomingContext, parentId, tabPageInfo.id, @@ -130,31 +138,24 @@ async function getTabs( false ); - warning(`Scan: Found tab "${tabs.nameOrTitle}". Processing tab's pages tree...`); + warning(`Scan: Found tab "${currentTab.nameOrTitle}". Processing tab's pages tree...`); - // Start new tree for this tab + // Initalize a structure for this tab + // TODO this probably dont need to be global layoutStrategy = new HierarchicalNamedLayoutStrategy(); - tabsPages = new Array(); - - // Create tab output folder - // const subfolderPath = options.markdownOutputPath.replace(/\/+$/, "") + '/' + tabs.nameOrTitle; - // await fs.mkdir(subfolderPath, { recursive: true }); + currentTabPages = new Array(); //TODO: this is static dont need to be looped layoutStrategy.setRootDirectoryForMarkdown(options.markdownOutputPath); // Process tab's pages group( - `Stage 1: walk children of tabs "${tabs.nameOrTitle}"` + `Stage 1: walk children of tabs "${currentTab.nameOrTitle}"` ); - await getPagesRecursively(options, "", options.rootPage, tabs.pageId, 0); - logDebug("getPagesRecursively", JSON.stringify(tabsPages, null, 2)); - info(`Found ${tabsPages.length} pages`); - endGroup(); - group( - `Stage 2: convert ${tabsPages.length} Notion pages to markdown and save locally...` - ); - await outputPages(options, config, tabsPages, allTabsPages); + await getTabsPagesRecursively(options, "", options.rootPage, currentTab.pageId, 0); + logDebug("getPagesRecursively", JSON.stringify(currentTabPages, null, 2)); + info(`Found ${currentTabPages.length} pages`); + allTabsPages[currentTab.nameOrTitle.toLowerCase()] = currentTabPages; endGroup(); } @@ -175,7 +176,7 @@ async function getTabs( // getPagesRecursively navigates the root page and iterates over each page within it, // treating each as an independent tree structure. It constructs a folder structure of pages for sidebar organization, // preserving the hierarchical order set in Notion. -async function getPagesRecursively( +async function getTabsPagesRecursively( options: DocuNotionOptions, incomingContext: string, parentId: string, @@ -218,12 +219,11 @@ async function getPagesRecursively( // Forward Category's index.md and push it into the pages array currentPage.layoutContext = layoutContext; - tabsPages.push(currentPage); - allTabsPages.push(currentPage); + currentTabPages.push(currentPage); // Recursively process child pages and page links for (const childPageInfo of pageInfo.childPageIdsAndOrder) { - await getPagesRecursively( + await getTabsPagesRecursively( options, layoutContext, currentPage.pageId, @@ -232,17 +232,7 @@ async function getPagesRecursively( ); } for (const linkPageInfo of pageInfo.linksPageIdsAndOrder) { - tabsPages.push( - await fromPageId( - layoutContext, - currentPage.pageId, - linkPageInfo.id, - linkPageInfo.order, - false, - true - ) - ); - allTabsPages.push( + currentTabPages.push( await fromPageId( layoutContext, currentPage.pageId, @@ -269,9 +259,10 @@ async function getPagesRecursively( ); for (const childPageInfo of pageInfo.childPageIdsAndOrder) { - await getPagesRecursively( + await getTabsPagesRecursively( options, layoutContext, + currentPage.pageId, childPageInfo.id, childPageInfo.order, @@ -279,17 +270,7 @@ async function getPagesRecursively( } for (const linkPageInfo of pageInfo.linksPageIdsAndOrder) { - tabsPages.push( - await fromPageId( - layoutContext, - currentPage.pageId, - linkPageInfo.id, - linkPageInfo.order, - false, - true - ) - ); - allTabsPages.push( + currentTabPages.push( await fromPageId( layoutContext, currentPage.pageId, @@ -305,8 +286,7 @@ async function getPagesRecursively( // Case: A simple content page else if (pageInfo.hasContent) { warning(`Scan: Page "${currentPage.nameOrTitle}" is a simple content page.`); - tabsPages.push(currentPage); - allTabsPages.push(currentPage); + currentTabPages.push(currentPage); } // Case: Empty pages and undefined ones @@ -479,53 +459,44 @@ export function numberChildrenIfNumberedList( async function outputPages( options: DocuNotionOptions, config: IDocuNotionConfig, - tabsPages: Array, - allTabsPages: Array + allTabsPages: Record ) { const context: IDocuNotionContext = { - config: config, - layoutStrategy: layoutStrategy, - options: options, - getBlockChildren: getBlockChildren, - notionToMarkdown: notionToMarkdown, - directoryContainingMarkdown: "", // this changes with each page - relativeFilePathToFolderContainingPage: "", // this changes with each page - convertNotionLinkToLocalDocusaurusLink: (url: string) => - convertInternalUrl(context, url), - tabsPages: tabsPages, - allTabsPages: allTabsPages, - counts: counts, // review will this get copied or pointed to? + config, + layoutStrategy, + options, + getBlockChildren, + notionToMarkdown, + directoryContainingMarkdown: "", + relativeFilePathToFolderContainingPage: "", + convertNotionLinkToLocalDocusaurusLink: (url: string) => convertInternalUrl(context, url), + allTabsPages, + currentTab: "", + counts, imports: [], - }; - for (const page of tabsPages) { - const mdPath = layoutStrategy.getPathForPage(page, ".mdx"); - - // most plugins should not write to disk, but those handling image files need these paths - context.directoryContainingMarkdown = Path.dirname(mdPath); - // TODO: This needs clarifying: getLinkPathForPage() is about urls, but - // downstream images.ts is using it as a file system path - context.relativeFilePathToFolderContainingPage = Path.dirname( - layoutStrategy.getLinkPathForPage(page) - ); - if ( - page.type === PageType.DatabasePage && - context.options.statusTag != "*" && - page.status !== context.options.statusTag - ) { - verbose( - `Skipping page because status is not '${context.options.statusTag}': ${page.nameOrTitle}` - ); - // TODO: count need to be reset for each loop otherwise moot - ++context.counts.skipped_because_status; - } else { - //TODO: config no longer needs to be passed now that it is part of context - const markdown = await getMarkdownForPage(config, context, page); - writePage(page, markdown); + for (const tab in allTabsPages) { + const tabPages = allTabsPages[tab]; + context.currentTab = tab; + context.counts.skipped_because_status = 0; + + for (const page of tabPages) { + const mdPath = layoutStrategy.getPathForPage(page, ".mdx"); + context.directoryContainingMarkdown = Path.dirname(mdPath); + context.relativeFilePathToFolderContainingPage = Path.dirname(layoutStrategy.getLinkPathForPage(page)); + + if (page.type === PageType.DatabasePage && context.options.statusTag != "*" && page.status !== context.options.statusTag) { + verbose(`Skipping page because status is not '${context.options.statusTag}': ${page.nameOrTitle}`); + ++context.counts.skipped_because_status; + } else { + const markdown = await getMarkdownForPage(context, page); + writePage(page, markdown); + } } - } - info(`Finished processing ${tabsPages.length} pages`); - info(JSON.stringify(counts)); + info(`Finished processing ${tab}`); + // TODO counts needs refactoring (mixing up total per tab and total all tabs) + info(JSON.stringify(counts.skipped_because_status)); + } } \ No newline at end of file diff --git a/src/transform.ts b/src/transform.ts index d25abbf..baeed2f 100644 --- a/src/transform.ts +++ b/src/transform.ts @@ -10,7 +10,6 @@ import { NotionBlock } from "./types"; import { executeWithRateLimitAndRetries } from "./pull"; export async function getMarkdownForPage( - config: IDocuNotionConfig, context: IDocuNotionContext, page: NotionPage ): Promise { @@ -55,7 +54,7 @@ export async function getMarkdownForPage( return true; }); - const body = await getMarkdownFromNotionBlocks(context, config, filteredBlocks); + const body = await getMarkdownFromNotionBlocks(context, context.config, filteredBlocks); const frontmatter = getFrontMatter(page); // todo should be a plugin return `${frontmatter}\n${body}`; } @@ -232,7 +231,7 @@ function doLinkFixes( const originalLinkMarkdown = match[0]; verbose( - `Link parsing: Checking "${originalLinkMarkdown}"` + `Checking "${originalLinkMarkdown}"` ); // We only use the first plugin that matches and makes a change to the link. @@ -241,7 +240,7 @@ function doLinkFixes( config.plugins.some(plugin => { if (!plugin.linkModifier) return false; if (plugin.linkModifier.match.exec(originalLinkMarkdown) === null) { - verbose(`Link parsing: [${plugin.name}] Did not match this url`); + verbose(`[${plugin.name}] Did not match this url`); return false; } const newMarkdown = plugin.linkModifier.convert( @@ -252,11 +251,11 @@ function doLinkFixes( if (newMarkdown !== originalLinkMarkdown) { markdown = markdown.replace(originalLinkMarkdown, newMarkdown); verbose( - `Link parsing: [${plugin.name}] Converted "${originalLinkMarkdown}" to "${newMarkdown}"` + `[${plugin.name}] Converted "${originalLinkMarkdown}" to "${newMarkdown}"` ); return true; // the first plugin that matches and does something wins } else { - verbose(`Link parsing: [${plugin.name}] URL unchanged`); + verbose(`[${plugin.name}] URL unchanged`); return false; } }); From 756d4a72b760f89a6cfad05f78a9e029f627cde5 Mon Sep 17 00:00:00 2001 From: kmlbgn Date: Thu, 7 Mar 2024 20:51:55 +0800 Subject: [PATCH 2/2] feat: renaming to Nocusaurus --- .github/ISSUE_TEMPLATE/conversion-problem.md | 2 +- docu-notion.config.ts | 4 ++-- package-lock.json | 6 +++--- package.json | 6 +++--- src/NotionPage.ts | 2 +- src/plugins/README.md | 2 +- src/plugins/VideoTransformer.ts | 2 +- src/plugins/externalLinks.ts | 2 +- src/pull.ts | 4 ++-- src/run.ts | 10 +++++----- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/conversion-problem.md b/.github/ISSUE_TEMPLATE/conversion-problem.md index 6e9399a..7e54248 100644 --- a/.github/ISSUE_TEMPLATE/conversion-problem.md +++ b/.github/ISSUE_TEMPLATE/conversion-problem.md @@ -12,7 +12,7 @@ A clear and concise description of what the bug is. ** Reproduction Steps ** -URL of a source Notion page that, when docu-notion accesses or converts it, shows the problem: +URL of a source Notion page that, when Nocusaurus accesses or converts it, shows the problem: At least one of these: 1) URL of a docusuarus site showing the resulting page diff --git a/docu-notion.config.ts b/docu-notion.config.ts index 35c3c92..528963c 100644 --- a/docu-notion.config.ts +++ b/docu-notion.config.ts @@ -1,5 +1,5 @@ -/* This file is only used when testing docu-notion itself, not when it is used as a library. - E.g., if you run `npm run pull-test-tagged`, docu-notion will read this file and use it to configure itself, +/* This file is only used when testing Nocusaurus itself, not when it is used as a library. + E.g., if you run `npm run pull-test-tagged`, Nocusaurus will read this file and use it to configure itself, using these example plugins. */ diff --git a/package-lock.json b/package-lock.json index 053e9f7..ec72692 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,10 +1,10 @@ { - "name": "docu-notion-kira", + "name": "nocusaurus", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "docu-notion-kira", + "name": "nocusaurus", "license": "MIT", "dependencies": { "@notionhq/client": "2.2.3", @@ -24,7 +24,7 @@ "ts-node": "^10.2.1" }, "bin": { - "docu-notion-kira": "dist/index.js" + "nocusaurus": "dist/index.js" }, "devDependencies": { "@types/fs-extra": "^9.0.13", diff --git a/package.json b/package.json index 51bc8bb..16bb3db 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "vite": "^4.2.1", "vitest": "^0.30.1" }, - "name": "docu-notion-kira", + "name": "nocusaurus", "description": "Download Notion pages as markdown and image files, preserving hierarchy and enabling workflow properties. Works with Docusaurus.", "license": "MIT", "author": { @@ -82,9 +82,9 @@ "documentation" ], "bugs": { - "url": "https://github.com/KiraCore/docu-notion-kira/issues" + "url": "https://github.com/KiraCore/nocusaurus/issues" }, - "homepage": "https://github.com/KiraCore/docu-notion-kira#readme", + "homepage": "https://github.com/KiraCore/nocusaurus#readme", "main": "./dist/index.js", "bin": "dist/index.js", "files": [ diff --git a/src/NotionPage.ts b/src/NotionPage.ts index 1b48773..1f90731 100644 --- a/src/NotionPage.ts +++ b/src/NotionPage.ts @@ -5,7 +5,7 @@ import { parseLinkId } from "./plugins/internalLinks"; import { ListBlockChildrenResponseResults } from "notion-to-md/build/types"; // Notion has 2 kinds of pages: a normal one which is just content, and what I'm calling a "database page", which has whatever properties you put on it. -// docu-notion supports the later via links from outline pages. That is, you put the database pages in a database, then separately, in the outline, you +// Nocusaurus supports the later via links from outline pages. That is, you put the database pages in a database, then separately, in the outline, you // create pages for each node of the outline and then add links from those to the database pages. In this way, we get the benefits of database // pages (metadata, workflow, etc) and also normal pages (order, position in the outline). export enum PageType { diff --git a/src/plugins/README.md b/src/plugins/README.md index 71b13ab..18ff7ab 100644 --- a/src/plugins/README.md +++ b/src/plugins/README.md @@ -1,6 +1,6 @@ # Plugins (Advanced & Experimental) -If your project needs some processing that docu-notion doesn't already provide, you can provide a plugin that does it. If there is call for it, we'll add more documentation in the future. But for now, here's the steps: +If your project needs some processing that Nocusaurus doesn't already provide, you can provide a plugin that does it. If there is call for it, we'll add more documentation in the future. But for now, here's the steps: 1. Add a `docu-notion.config.ts` to the root level of your project directory. 1. Add something like this: diff --git a/src/plugins/VideoTransformer.ts b/src/plugins/VideoTransformer.ts index a07c17d..ce2e177 100644 --- a/src/plugins/VideoTransformer.ts +++ b/src/plugins/VideoTransformer.ts @@ -28,7 +28,7 @@ export const standardVideoTransformer: IPlugin = { warning( `[standardVideoTransformer] Found Notion "video" block with type ${JSON.stringify( (video as any).type - )}. The best docu-notion can do for now is ignore it.` + )}. The best Nocusaurus can do for now is ignore it.` ); return ""; break; diff --git a/src/plugins/externalLinks.ts b/src/plugins/externalLinks.ts index 16f933e..4e569a1 100644 --- a/src/plugins/externalLinks.ts +++ b/src/plugins/externalLinks.ts @@ -19,7 +19,7 @@ export const standardExternalLinkConversion: IPlugin = { if (label === "bookmark") { const replacement = `[${url}](${url})`; warning( - `Link parsing: [ExternalLinkPlugin] Found Notion "Bookmark" link. In Notion this would show as an embed. The best docu-notion can do at the moment is replace "Bookmark" with the actual URL: ${replacement}` + `Link parsing: [ExternalLinkPlugin] Found Notion "Bookmark" link. In Notion this would show as an embed. The best Nocusaurus can do at the moment is replace "Bookmark" with the actual URL: ${replacement}` ); return replacement; } diff --git a/src/pull.ts b/src/pull.ts index c3ff43f..669c27a 100644 --- a/src/pull.ts +++ b/src/pull.ts @@ -77,7 +77,7 @@ export async function notionPull(options: DocuNotionOptions): Promise { }); } catch (e: any) { error( - `docu-notion could not retrieve the root page from Notion. \r\na) Check that the root page id really is "${ + `Nocusaurus could not retrieve the root page from Notion. \r\na) Check that the root page id really is "${ options.rootPage }".\r\nb) Check that your Notion API token (the "Integration Secret") is correct. It starts with "${ optionsForLogging.notionToken @@ -398,7 +398,7 @@ async function getBlockChildren(id: string): Promise { if (overallResult?.results?.some(b => !isFullBlock(b))) { error( - `The Notion API returned some blocks that were not full blocks. docu-notion does not handle this yet. Please report it.` + `The Notion API returned some blocks that were not full blocks. Nocusaurus does not handle this yet. Please report it.` ); exit(1); } diff --git a/src/run.ts b/src/run.ts index ffed2bf..2a75a05 100644 --- a/src/run.ts +++ b/src/run.ts @@ -8,7 +8,7 @@ import path from "path"; export async function run(): Promise { const pkg = require("../package.json"); // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - console.log(`docu-notion version ${pkg.version}`); + console.log(`Nocusaurus version ${pkg.version}`); program.name("docu-notion").description(""); program.usage("-n -r [options]"); @@ -23,12 +23,12 @@ export async function run(): Promise { ) .option( "-m, --markdown-output-path ", - "Root of the hierarchy for md files. WARNING: docu-notion will delete files from this directory. Note also that if it finds localized images, it will create an i18n/ directory as a sibling.", + "Root of the hierarchy for md files. WARNING: Nocusaurus will delete files from this directory. Note also that if it finds localized images, it will create an i18n/ directory as a sibling.", "./tabs" ) .option( "--css-output-directory ", - "docu-notion has a docu-notion-styles.css file that you will need to use to get things like notion columns to look right. This option specifies where that file should be copied to.", + "Nocusaurus has a docu-notion-styles.css file that you will need to use to get things like notion columns to look right. This option specifies where that file should be copied to.", "./css" ) .option( @@ -74,10 +74,10 @@ export async function run(): Promise { let pathToCss = ""; try { pathToCss = require.resolve( - "docu-notion-kira/dist/docu-notion-styles.css" + "nocusaurus/dist/docu-notion-styles.css" ); } catch (e) { - // when testing from the docu-notion project itself: + // when testing from the Nocusaurus project itself: pathToCss = "./src/css/docu-notion-styles.css"; } // make any missing parts of the path exist