Added markdown rendering

This commit is contained in:
Magnus Åhall 2026-05-15 08:22:43 +02:00
parent 26ca510785
commit 5a0340c226
172 changed files with 12198 additions and 8338 deletions

View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 @UziTech
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View file

@ -0,0 +1,160 @@
# marked-token-position
Add `position` field for each token.
```ts
interface Position {
/**
* Positions for each line of the token. LinePositions will not include the newline character for the line.
*/
lines: LinePosition[]
/**
* Position at the beginning of token
*/
start: PositionFields;
/**
* Position at the end of token
*/
end: PositionFields;
}
interface LinePosition {
/**
* Position at the beginning of line
*/
start: PositionFields;
/**
* Position at the end of line. Will not include the newline character.
*/
end: PositionFields;
}
interface PositionFields {
/**
* Number of characters from the beginning of the markdown string
*/
offset: number;
/**
* Line number of the token. Starts at line 0.
*/
line: number;
/**
* Column number of the token. Starts at column 0.
*/
column: number;
}
```
# Usage
## Extension
```js
import {Marked} from "marked";
import markedTokenPosition from "marked-token-position";
// or UMD script
// <script src="https://cdn.jsdelivr.net/npm/marked/lib/marked.umd.js"></script>
// <script src="https://cdn.jsdelivr.net/npm/marked-token-position/lib/index.umd.js"></script>
// const Marked = marked.Marked;
const marked = new Marked();
function anotherExtension {
return {
walkTokens(token) {
// token has `position` field
}
hooks: {
processAllTokens(tokens) {
// tokens have `position` field
}
}
};
}
marked.use(anotherExtension(), markedTokenPosition());
marked.parse("# example markdown");
```
The `position` field will be added to the tokens so any other extension can
use the `position` field in a `walkTokens` function or `processAllTokens` hook.
> [!CAUTION]
> The `processAllTokens` hook is used by this extension so any other extension
> using `processAllTokens` that requires the `position` field must be added
> before this extension because marked calls the `processAllTokens` hooks in
> reverse order.
The tokens will look like:
```json
[
{
"type": "heading",
"raw": "# example markdown",
"depth": 1,
"text": "example markdown",
"tokens": [
{
"type": "text",
"raw": "example markdown",
"text": "example markdown",
"escaped": false,
"position": {
"start": {
"offset": 2,
"line": 0,
"column": 2
},
"end": {
"offset": 18,
"line": 0,
"column": 18
}
}
}
],
"position": {
"start": {
"offset": 0,
"line": 0,
"column": 0
},
"end": {
"offset": 18,
"line": 0,
"column": 18
}
}
}
]
```
## addTokenPositions
Calling `marked.lexer()` will not add the `position` field with the extension
since the extension is only called on `marked.parse()` and `marked.parseInline()`.
An `addTokenPositions` function is exported to add the `position` field to the
tokens returned by `marked.lexer()`.
```js
import {Marked} from "marked";
import {addTokenPositions} from "marked-token-position";
// or UMD script
// <script src="https://cdn.jsdelivr.net/npm/marked/lib/marked.umd.js"></script>
// <script src="https://cdn.jsdelivr.net/npm/marked-token-position/lib/index.umd.js"></script>
// const Marked = marked.Marked;
// const addTokenPositions = markedTokenPosition.addTokenPositions;
const marked = new Marked();
const tokens = marked.lexer("# example markdown");
addTokenPositions(tokens);
// tokens now have a `position` field
```

View file

@ -0,0 +1,59 @@
// Generated by dts-bundle-generator v9.5.1
import { MarkedExtension, Token, Tokens } from 'marked';
export interface TokenWithPosition extends Tokens.Generic {
position: Position;
}
export interface Position {
/**
* Positions for each line of the token. LinePositions will not include the newline character for the line.
*/
lines: LinePosition[];
/**
* Position at the beginning of token
*/
start: PositionFields;
/**
* Position at the end of token
*/
end: PositionFields;
}
export interface LinePosition {
/**
* Position at the beginning of line
*/
start: PositionFields;
/**
* Position at the end of line. Will not include the newline character.
*/
end: PositionFields;
}
export interface PositionFields {
/**
* Number of characters from the beginning of the markdown string
*/
offset: number;
/**
* Line number of the token. Starts at line 0.
*/
line: number;
/**
* Column number of the token. Starts at column 0.
*/
column: number;
}
/**
* Add position field to tokens
*/
export declare function addTokenPositions(tokens: Token[]): TokenWithPosition[];
/**
* Marked extension to add position field to tokens
*/
declare function _default(options?: {}): MarkedExtension;
export {
_default as default,
};
export {};

View file

@ -0,0 +1,6 @@
function g(u){let i=u.map(r=>r.raw).join("");return h(u,0,0,0,i).tokens}function b(u={}){return{hooks:{processAllTokens(i){return g(i)}}}}function h(u,i,r,f,l){for(let s of u){let n=s,a=T(i,r,f,l,n.raw);if(n.position=a,n.tokens&&h(n.tokens,i,r,f,l),n.childTokens){let c=i,t=r,e=f,d=l;for(let k of n.childTokens){let o=h(n[k],c,t,e,d);c=o.offset,t=o.line,e=o.column,d=o.markdown}}if(n.type==="list"&&h(n.items,i,r,f,l),n.type==="table"){let c=i,t=r,e=f,d=l;for(let k of n.header){let o=h(k.tokens,c,t,e,d);c=o.offset,t=o.line,e=o.column,d=o.markdown}for(let k of n.rows)for(let o of k){let P=h(o.tokens,c,t,e,d);c=P.offset,t=P.line,e=P.column,d=P.markdown}}let m=a.end.offset-i;i=a.end.offset,r=a.end.line,f=a.end.column,l=l.slice(m)}return{tokens:u,offset:i,line:r,column:f,markdown:l}}function T(u,i,r,f,l){let s=[],n=l.split(`
`),a=f.split(`
`);n:for(let t=0;t<=a.length-n.length;t++){s=[];for(let e=0;e<n.length;e++){let d=a[t+e],k=n[e],o=d.indexOf(k);if(o===-1)continue n;let P=a.slice(0,t+e).join(`
`)+(t+e>0?`
`:""),x={offset:u+P.length+o,line:i+t+e,column:(t+e===0?r:0)+o},p={offset:x.offset+k.length,line:x.line,column:x.column+k.length};s.push({start:x,end:p})}break}if(s.length===0)throw new Error(`Cannot find ${JSON.stringify(l)} in ${JSON.stringify(f)}`);let m=s[0].start,c=s.at(-1).end;return s.length>1&&s.at(-1).start.offset===c.offset&&(s=s.slice(0,-1)),{lines:s,start:m,end:c}}export{g as addTokenPositions,b as default};
//# sourceMappingURL=index.esm.js.map

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,9 @@
(function(g,f){if(typeof exports=="object"&&typeof module<"u"){module.exports=f()}else if("function"==typeof define && define.amd){define("markedTokenPosition",f)}else {g["markedTokenPosition"]=f()}}(typeof globalThis < "u" ? globalThis : typeof self < "u" ? self : this,function(){var exports={};var __exports=exports;var module={exports};
var m=Object.defineProperty;var O=Object.getOwnPropertyDescriptor;var y=Object.getOwnPropertyNames;var C=Object.prototype.hasOwnProperty;var F=(e,n)=>()=>(e&&(n=e(e=0)),n);var M=(e,n)=>{for(var o in n)m(e,o,{get:n[o],enumerable:!0})},j=(e,n,o,f)=>{if(n&&typeof n=="object"||typeof n=="function")for(let t of y(n))!C.call(e,t)&&t!==o&&m(e,t,{get:()=>n[t],enumerable:!(f=O(n,t))||f.enumerable});return e};var b=e=>j(m({},"__esModule",{value:!0}),e);var T={};M(T,{addTokenPositions:()=>L,default:()=>E});function L(e){let n=e.map(o=>o.raw).join("");return x(e,0,0,0,n).tokens}function E(e={}){return{hooks:{processAllTokens(n){return L(n)}}}}function x(e,n,o,f,t){for(let c of e){let i=c,a=S(n,o,f,t,i.raw);if(i.position=a,i.tokens&&x(i.tokens,n,o,f,t),i.childTokens){let d=n,r=o,s=f,u=t;for(let k of i.childTokens){let l=x(i[k],d,r,s,u);d=l.offset,r=l.line,s=l.column,u=l.markdown}}if(i.type==="list"&&x(i.items,n,o,f,t),i.type==="table"){let d=n,r=o,s=f,u=t;for(let k of i.header){let l=x(k.tokens,d,r,s,u);d=l.offset,r=l.line,s=l.column,u=l.markdown}for(let k of i.rows)for(let l of k){let P=x(l.tokens,d,r,s,u);d=P.offset,r=P.line,s=P.column,u=P.markdown}}let p=a.end.offset-n;n=a.end.offset,o=a.end.line,f=a.end.column,t=t.slice(p)}return{tokens:e,offset:n,line:o,column:f,markdown:t}}function S(e,n,o,f,t){let c=[],i=t.split(`
`),a=f.split(`
`);n:for(let r=0;r<=a.length-i.length;r++){c=[];for(let s=0;s<i.length;s++){let u=a[r+s],k=i[s],l=u.indexOf(k);if(l===-1)continue n;let P=a.slice(0,r+s).join(`
`)+(r+s>0?`
`:""),h={offset:e+P.length+l,line:n+r+s,column:(r+s===0?o:0)+l},w={offset:h.offset+k.length,line:h.line,column:h.column+k.length};c.push({start:h,end:w})}break}if(c.length===0)throw new Error(`Cannot find ${JSON.stringify(t)} in ${JSON.stringify(f)}`);let p=c[0].start,d=c.at(-1).end;return c.length>1&&c.at(-1).start.offset===d.offset&&(c=c.slice(0,-1)),{lines:c,start:p,end:d}}var g=F(()=>{"use strict"});module.exports=(g(),b(T)).default;module.exports.addTokenPositions=(g(),b(T)).addTokenPositions;
if(__exports != exports)module.exports = exports;return module.exports}));
//# sourceMappingURL=index.umd.js.map

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,66 @@
{
"name": "marked-token-position",
"version": "2.0.2",
"description": "marked extension template",
"main": "./lib/index.esm.js",
"module": "./lib/index.esm.js",
"browser": "./lib/index.umd.js",
"type": "module",
"keywords": [
"marked",
"extension"
],
"files": [
"lib/",
"src/"
],
"exports": {
".": {
"typescript": "./src/index.ts",
"types": "./lib/index.d.ts",
"default": "./lib/index.esm.js"
}
},
"scripts": {
"build": "npm run build:esbuild && npm run build:types",
"build:esbuild": "node esbuild.config.js",
"build:types": "tsc && dts-bundle-generator --export-referenced-types --project tsconfig.json -o lib/index.d.ts src/index.ts",
"format": "eslint --fix",
"lint": "eslint",
"test": "npm run build:esbuild && node --experimental-transform-types ./spec/test.config.js",
"test:cover": "npm run build:esbuild && node --experimental-transform-types --experimental-test-coverage ./spec/test.config.js -- --cover",
"test:only": "npm run build:esbuild && node --experimental-transform-types ./spec/test.config.js -- --only",
"test:types": "npm run build:types && tsc --project tsconfig-test-types.json && attw -P --entrypoints . --profile esm-only",
"test:update": "npm run build:esbuild && node --experimental-transform-types --test-update-snapshots ./spec/test.config.js"
},
"repository": {
"type": "git",
"url": "git+https://github.com/UziTech/marked-token-position.git"
},
"author": "Tony Brix <Tony@Brix.ninja> (https://Tony.Brix.ninja)",
"license": "MIT",
"bugs": {
"url": "https://github.com/UziTech/marked-token-position/issues"
},
"homepage": "https://github.com/UziTech/marked-token-position#readme",
"peerDependencies": {
"marked": ">=16.2.0 <19"
},
"devDependencies": {
"@arethetypeswrong/cli": "^0.18.2",
"@markedjs/eslint-config": "^1.0.14",
"@semantic-release/changelog": "^6.0.3",
"@semantic-release/commit-analyzer": "^13.0.1",
"@semantic-release/git": "^10.0.1",
"@semantic-release/github": "^12.0.6",
"@semantic-release/npm": "^13.1.5",
"@semantic-release/release-notes-generator": "^14.1.0",
"dts-bundle-generator": "^9.5.1",
"esbuild": "^0.28.0",
"esbuild-plugin-umd-wrapper": "^3.0.0",
"eslint": "^10.2.0",
"marked": "^18.0.0",
"semantic-release": "^25.0.3",
"typescript": "^6.0.2"
}
}

View file

@ -0,0 +1,192 @@
/* node:coverage ignore next */
import type { MarkedExtension, Token, Tokens } from 'marked';
export interface TokenWithPosition extends Tokens.Generic {
position: Position;
}
interface Position {
/**
* Positions for each line of the token. LinePositions will not include the newline character for the line.
*/
lines: LinePosition[]
/**
* Position at the beginning of token
*/
start: PositionFields;
/**
* Position at the end of token
*/
end: PositionFields;
}
interface LinePosition {
/**
* Position at the beginning of line
*/
start: PositionFields;
/**
* Position at the end of line. Will not include the newline character.
*/
end: PositionFields;
}
interface PositionFields {
/**
* Number of characters from the beginning of the markdown string
*/
offset: number;
/**
* Line number of the token. Starts at line 0.
*/
line: number;
/**
* Column number of the token. Starts at column 0.
*/
column: number;
}
/**
* Add position field to tokens
*/
export function addTokenPositions(tokens: Token[]) {
const markdown = tokens.map(token => token.raw).join('');
return addPosition(tokens, 0, 0, 0, markdown).tokens;
}
/**
* Marked extension to add position field to tokens
*/
export default function(options = {}): MarkedExtension {
return {
hooks: {
processAllTokens(tokens) {
return addTokenPositions(tokens);
},
},
};
}
function addPosition(tokens: Token[], offset: number, line: number, column: number, markdown: string) {
for (const token of tokens) {
const genericToken = token as Tokens.Generic;
const position = getPosition(offset, line, column, markdown, genericToken.raw);
genericToken.position = position;
if (genericToken.tokens) {
addPosition(genericToken.tokens, offset, line, column, markdown);
}
if (genericToken.childTokens) {
let nextOffset = offset;
let nextLine = line;
let nextColumn = column;
let nextMarkdown = markdown;
for (const childToken of genericToken.childTokens) {
const nextPosition = addPosition(genericToken[childToken], nextOffset, nextLine, nextColumn, nextMarkdown);
nextOffset = nextPosition.offset;
nextLine = nextPosition.line;
nextColumn = nextPosition.column;
nextMarkdown = nextPosition.markdown;
}
}
if (genericToken.type === 'list') {
addPosition(genericToken.items, offset, line, column, markdown);
}
if (genericToken.type === 'table') {
let nextOffset = offset;
let nextLine = line;
let nextColumn = column;
let nextMarkdown = markdown;
for (const headerCell of genericToken.header) {
const nextPosition = addPosition(headerCell.tokens, nextOffset, nextLine, nextColumn, nextMarkdown);
nextOffset = nextPosition.offset;
nextLine = nextPosition.line;
nextColumn = nextPosition.column;
nextMarkdown = nextPosition.markdown;
}
for (const row of genericToken.rows) {
for (const rowCell of row) {
const nextPosition = addPosition(rowCell.tokens, nextOffset, nextLine, nextColumn, nextMarkdown);
nextOffset = nextPosition.offset;
nextLine = nextPosition.line;
nextColumn = nextPosition.column;
nextMarkdown = nextPosition.markdown;
}
}
}
const deltaOffset = position.end.offset - offset;
offset = position.end.offset;
line = position.end.line;
column = position.end.column;
markdown = markdown.slice(deltaOffset);
}
return {
tokens: tokens as TokenWithPosition[],
offset,
line,
column,
markdown,
};
}
function getPosition(offset: number, line: number, column: number, markdown: string, raw: string): Position {
let lines: LinePosition[] = [];
const rawLines = raw.split('\n');
const markdownLines = markdown.split('\n');
// eslint-disable-next-line no-labels
md: for (let i = 0; i <= markdownLines.length - rawLines.length; i++) {
lines = [];
for (let j = 0; j < rawLines.length; j++) {
const markdownLine = markdownLines[i + j];
const rawLine = rawLines[j];
const lineStartOffset = markdownLine.indexOf(rawLine);
if (lineStartOffset === -1) {
// eslint-disable-next-line no-labels
continue md;
}
const beforeMarkdownLines = markdownLines.slice(0, i + j).join('\n') + (i + j > 0 ? '\n' : '');
const start = {
offset: offset + beforeMarkdownLines.length + lineStartOffset,
line: line + i + j,
column: (i + j === 0 ? column : 0) + lineStartOffset,
};
const end = {
offset: start.offset + rawLine.length,
line: start.line,
column: start.column + rawLine.length,
};
lines.push({
start,
end,
});
}
break;
}
/* node:coverage ignore next 4 */
if (lines.length === 0) {
// This shouldn't ever happen but if it does it would be nice to have a good error message
throw new Error(`Cannot find ${JSON.stringify(raw)} in ${JSON.stringify(markdown)}`);
}
const start = lines[0].start;
const end = lines.at(-1)!.end;
if (lines.length > 1 && lines.at(-1)!.start.offset === end.offset) {
lines = lines.slice(0, -1);
}
return {
lines,
start,
end,
};
}