From 720681fcbabc17e6134831c45d0d6eebde3c6c6a Mon Sep 17 00:00:00 2001 From: Katerina Skroumpelou Date: Tue, 6 Aug 2024 16:13:03 +0300 Subject: [PATCH] fix(nx-dev): clean up heading text (#27205) Fixing 3 issues when storing the page sections: 1. The heading of a node can be like this: `title: 'Angular Monorepo Tutorial - Part 1: Code Generation' description: In this tutorial you'll create a frontend-focused workspace with Nx`. We only want to keep the `title` part. 2. Some headings have `{% highlightColor="green" %}`. We want to remove that from the heading of the page section (and the link text) 3. There's an [error in `tags.json` where the path is incorrect](https://github.com/nrwl/nx/blob/master/docs/generated/manifests/tags.json#L126). We want to fix that on the fly, so that the embeddings script finds the correct pages. --- docs/generated/manifests/tags.json | 28 ++++---- .../generators/generate-manifests.ts | 10 ++- .../create-embeddings/src/main.mts | 68 +++++++++++++------ 3 files changed, 68 insertions(+), 38 deletions(-) diff --git a/docs/generated/manifests/tags.json b/docs/generated/manifests/tags.json index 1430e290af..9225539083 100644 --- a/docs/generated/manifests/tags.json +++ b/docs/generated/manifests/tags.json @@ -123,21 +123,21 @@ }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/run", + "file": "generated/packages/nx/documents/run", "id": "run", "name": "run", "path": "/nx-api/nx/documents/run" }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/run-many", + "file": "generated/packages/nx/documents/run-many", "id": "run-many", "name": "run-many", "path": "/nx-api/nx/documents/run-many" }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/affected", + "file": "generated/packages/nx/documents/affected", "id": "affected", "name": "affected", "path": "/nx-api/nx/documents/affected" @@ -230,14 +230,14 @@ }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/connect-to-nx-cloud", + "file": "generated/packages/nx/documents/connect-to-nx-cloud", "id": "connect-to-nx-cloud", "name": "connect-to-nx-cloud", "path": "/nx-api/nx/documents/connect-to-nx-cloud" }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/reset", + "file": "generated/packages/nx/documents/reset", "id": "reset", "name": "reset", "path": "/nx-api/nx/documents/reset" @@ -274,7 +274,7 @@ }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/dep-graph", + "file": "generated/packages/nx/documents/dep-graph", "id": "dep-graph", "name": "graph", "path": "/nx-api/nx/documents/dep-graph" @@ -339,7 +339,7 @@ }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/generate", + "file": "generated/packages/nx/documents/generate", "id": "generate", "name": "generate", "path": "/nx-api/nx/documents/generate" @@ -369,7 +369,7 @@ }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/migrate", + "file": "generated/packages/nx/documents/migrate", "id": "migrate", "name": "migrate", "path": "/nx-api/nx/documents/migrate" @@ -441,14 +441,14 @@ }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/format-check", + "file": "generated/packages/nx/documents/format-check", "id": "format-check", "name": "format:check", "path": "/nx-api/nx/documents/format-check" }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/format-write", + "file": "generated/packages/nx/documents/format-write", "id": "format-write", "name": "format:write", "path": "/nx-api/nx/documents/format-write" @@ -541,7 +541,7 @@ }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/release", + "file": "generated/packages/nx/documents/release", "id": "release", "name": "release", "path": "/nx-api/nx/documents/release" @@ -631,7 +631,7 @@ }, { "description": "The Nx Plugin for Angular contains executors, generators, and utilities for managing Angular applications and libraries within an Nx workspace. It provides: \n\n- Integration with libraries such as Storybook, Jest, ESLint, Tailwind CSS, Playwright and Cypress. \n\n- Generators to help scaffold code quickly (like: Micro Frontends, Libraries, both internal to your codebase and publishable to npm) \n\n- Single Component Application Modules (SCAMs) \n\n- NgRx helpers. \n\n- Utilities for automatic workspace refactoring.", - "file": "generated/packages/generated/packages/angular/documents/nx-devkit-angular-devkit", + "file": "generated/packages/angular/documents/nx-devkit-angular-devkit", "id": "nx-devkit-angular-devkit", "name": "Nx Devkit and Angular Devkit", "path": "/nx-api/angular/documents/nx-devkit-angular-devkit" @@ -783,7 +783,7 @@ }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/watch", + "file": "generated/packages/nx/documents/watch", "id": "watch", "name": "watch", "path": "/nx-api/nx/documents/watch" @@ -1113,7 +1113,7 @@ }, { "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", - "file": "generated/packages/generated/packages/nx/documents/connect-to-nx-cloud", + "file": "generated/packages/nx/documents/connect-to-nx-cloud", "id": "connect-to-nx-cloud", "name": "connect-to-nx-cloud", "path": "/nx-api/nx/documents/connect-to-nx-cloud" diff --git a/scripts/documentation/generators/generate-manifests.ts b/scripts/documentation/generators/generate-manifests.ts index 9318ba27ac..22f91ec4a0 100644 --- a/scripts/documentation/generators/generate-manifests.ts +++ b/scripts/documentation/generators/generate-manifests.ts @@ -183,11 +183,15 @@ function generateTags(manifests: Manifest[]) { Object.values(item.documents).forEach( (documentMetadata: DocumentMetadata) => { documentMetadata.tags.forEach((t: string) => { + const filePath = documentMetadata.file.startsWith( + 'generated/packages' + ) + ? documentMetadata.file + : ['generated', 'packages', documentMetadata.file].join('/'); + const tagData = { description: documentMetadata.description, - file: ['generated', 'packages', documentMetadata.file].join( - '/' - ), + file: filePath, id: documentMetadata.id, name: documentMetadata.name, path: documentMetadata.path, diff --git a/tools/documentation/create-embeddings/src/main.mts b/tools/documentation/create-embeddings/src/main.mts index 6ae28ee691..0bd69f50df 100644 --- a/tools/documentation/create-embeddings/src/main.mts +++ b/tools/documentation/create-embeddings/src/main.mts @@ -83,7 +83,9 @@ export function processMdxForSearch(content: string): ProcessedMdx { const [firstNode] = tree.children; const heading = - firstNode.type === 'heading' ? toString(firstNode) : undefined; + firstNode.type === 'heading' + ? removeTitleDescriptionFromHeading(toString(firstNode)) + : undefined; const slug = heading ? slugger.slug(heading) : undefined; return { @@ -320,28 +322,29 @@ async function generateEmbeddings() { const longer_heading = source !== 'community-plugins' - ? createLongerHeading(heading, url_partial) + ? removeTitleDescriptionFromHeading( + createLongerHeading(heading, url_partial) + ) : heading; - const { error: insertPageSectionError, data: pageSection } = - await supabaseClient - .from('nods_page_section') - .insert({ - page_id: page.id, - slug, - heading: - heading?.length && heading !== null && heading !== 'null' - ? heading - : longer_heading, - longer_heading, - content, - url_partial, - token_count: embeddingResponse.usage.total_tokens, - embedding: responseData.embedding, - }) - .select() - .limit(1) - .single(); + const { error: insertPageSectionError } = await supabaseClient + .from('nods_page_section') + .insert({ + page_id: page.id, + slug, + heading: + heading?.length && heading !== null && heading !== 'null' + ? heading + : longer_heading, + longer_heading, + content, + url_partial, + token_count: embeddingResponse.usage.total_tokens, + embedding: responseData.embedding, + }) + .select() + .limit(1) + .single(); if (insertPageSectionError) { throw insertPageSectionError; @@ -420,6 +423,7 @@ function getAllFilesWithItemList(data): WalkEntry[] { if (item.file && item.file.length > 0) { // the path is the relative path to the file within the nx repo // the url_partial is the relative path to the file within the docs site - under nx.dev + files.push({ path: `docs/${item.file}.md`, url_partial: item.path }); if (!identityMap[item.id]) { identityMap = { ...identityMap, [item.id]: item }; @@ -488,6 +492,28 @@ function createMarkdownForCommunityPlugins(): { }); } +function removeTitleDescriptionFromHeading( + inputString?: string +): string | null { + /** + * Heading node can be like this: + * title: 'Angular Monorepo Tutorial - Part 1: Code Generation' + * description: In this tutorial you'll create a frontend-focused workspace with Nx. + * + * We only want to keep the title part. + */ + if (!inputString) { + return null; + } + const titleMatch = inputString.match(/title:\s*(.+?)(?=\s*description:)/); + if (titleMatch) { + const title = titleMatch[1].trim(); + return title.replace(`{% highlightColor="green" %}`, '').trim(); + } else { + return inputString.replace(`{% highlightColor="green" %}`, '').trim(); + } +} + async function main() { await generateEmbeddings(); }