# mistune URL: /api/rendering/parsers/mistune/ Section: parsers -------------------------------------------------------------------------------- mistune - Bengal window.BENGAL_THEME_DEFAULTS = { appearance: 'dark', palette: 'snow-lynx' }; // Progressive Enhancement System Configuration window.Bengal = window.Bengal || {}; window.Bengal.enhanceBaseUrl = '/bengal/assets/js/enhancements'; window.Bengal.watchDom = true; window.Bengal.debug = false; (function () { try { var defaults = window.BENGAL_THEME_DEFAULTS || { appearance: 'system', palette: '' }; var defaultAppearance = defaults.appearance; if (defaultAppearance === 'system') { defaultAppearance = (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? 'dark' : 'light'; } var storedTheme = localStorage.getItem('bengal-theme'); var storedPalette = localStorage.getItem('bengal-palette'); var theme = storedTheme ? (storedTheme === 'system' ? defaultAppearance : storedTheme) : defaultAppearance; var palette = storedPalette ?? defaults.palette; document.documentElement.setAttribute('data-theme', theme); if (palette) { document.documentElement.setAttribute('data-palette', palette); } } catch (e) { document.documentElement.setAttribute('data-theme', 'light'); } })(); Skip to main content Magnifying Glass ESC Recent Clear Magnifying Glass No results for "" Try different keywords or check your spelling Start typing to search... ↑↓ Navigate ↵ Open ESC Close Powered by Lunr ᓚᘏᗢ Documentation Info About Arrow Clockwise Get Started Note Tutorials File Text Content Palette Theming Settings Building Starburst Extending Bookmark Reference Learning Tracks Releases Dev GitHub API Reference bengal CLI Magnifying Glass Search ⌘K Palette Appearance Chevron Down Mode Monitor System Sun Light Moon Dark Palette Snow Lynx Brown Bengal Silver Bengal Charcoal Bengal Blue Bengal List ᓚᘏᗢ Magnifying Glass Search X Close Documentation Info About Arrow Clockwise Get Started Note Tutorials File Text Content Palette Theming Settings Building Starburst Extending Bookmark Reference Learning Tracks Releases Dev GitHub API Reference bengal CLI Palette Appearance Chevron Down Mode Monitor System Sun Light Moon Dark Palette Snow Lynx Brown Bengal Silver Bengal Charcoal Bengal Blue Bengal API Reference __main__ bengal Caret Right Folder Analysis community_detection graph_analysis graph_reporting graph_visualizer knowledge_graph link_suggestions link_types page_rank path_analysis performance_advisor results Caret Right Folder Assets manifest pipeline Caret Right Folder Autodoc base config docstring_parser utils virtual_orchestrator Caret Right Folder Extractors cli openapi python Caret Right Folder Models cli common openapi python Caret Right Folder Cache asset_dependency_map cache_store cacheable compression dependency_tracker page_discovery_cache query_index query_index_registry taxonomy_index utils Caret Right Folder Build Cache autodoc_tracking core file_tracking fingerprint parsed_content_cache rendered_output_cache taxonomy_index_mixin validation_cache Caret Right Folder Indexes author_index category_index date_range_index section_index Caret Right Folder Cli __main__ base site_templates utils Caret Right Folder Commands assets build clean collections config debug explain fix health init perf project serve site skeleton sources theme utils validate Caret Right Folder Graph __main__ bridges communities orphans pagerank report suggest Caret Right Folder New config presets scaffolds site wizard Caret Right Folder Helpers cli_app_loader cli_output config_validation error_handling menu_config metadata progress site_loader traceback validation Caret Right Folder Skeleton hydrator schema Caret Right Folder Templates base registry Caret Right Folder Blog template Caret Right Folder Changelog template Caret Right Folder Default template Caret Right Folder Docs template Caret Right Folder Landing template Caret Right Folder Portfolio template Caret Right Folder Resume template Caret Right Folder Collections errors loader schemas validator Caret Right Folder Config defaults deprecation directory_loader env_overrides environment feature_mappings hash loader merge origin_tracker validators Caret Right Folder Content Layer entry loaders manager source Caret Right Folder Sources github local notion rest Caret Right Folder Content Types base registry strategies Caret Right Folder Core build_context cascade_engine menu section theme Caret Right Folder Asset asset_core css_transforms Caret Right Folder Page computed content metadata navigation operations page_core proxy relationships utils Caret Right Folder Site core data discovery factories page_caches properties section_registry theme Caret Right Folder Debug base config_inspector content_migrator delta_analyzer dependency_visualizer explainer incremental_debugger models reporter shortcode_sandbox Caret Right Folder Discovery asset_discovery content_discovery Caret Right Folder Fonts downloader generator Caret Right Folder Health autofix base health_check report Caret Right Folder Linkcheck async_checker ignore_policy internal_checker models orchestrator Caret Right Folder Validators anchors assets cache config connectivity cross_ref fonts links menu navigation output performance rendering rss sitemap taxonomy tracks Caret Right Folder Directives analysis checkers constants Caret Right Folder Orchestration asset content full_to_incremental incremental menu postprocess related_posts render section static streaming taxonomy Caret Right Folder Postprocess html_output redirects rss sitemap special_pages Caret Right Folder Output Formats index_generator json_generator llm_generator lunr_index_generator txt_generator utils Caret Right Folder Rendering api_doc_enhancer asset_extractor errors jinja_utils link_transformer link_validator pygments_cache renderer template_context template_profiler validator Caret Right Folder Parsers base factory mistune native_html pygments_patch python_markdown Caret Right Folder Pipeline core output thread_local toc transforms Caret Right Folder Plugins badges cross_references inline_icon term variable_substitution Caret Right Folder Directives _icons admonitions badge base button cache cards checklist code_tabs container contracts data_table dropdown embed errors example_label fenced figure glossary icon include list_table literalinclude marimo navigation options rubric steps tabs target term terminal tokens utils validator video Caret Right Folder Template Engine asset_url core environment manifest menu url_helpers Caret Right Folder Template Functions advanced_collections advanced_strings autodoc collections content crossref data dates debug files get_page i18n icons images math_functions navigation pagination_helpers seo strings tables taxonomies theme urls Caret Right Folder Server build_handler component_preview constants dev_server live_reload pid_manager reload_controller request_handler request_logger resource_manager utils Caret Right Folder Services validation Caret Right Folder Themes config Caret Right Folder Utils atomic_write autodoc build_context build_stats build_summary cli_output css_minifier dates dotdict error_handlers file_io file_lock hashing incremental_constants js_bundler live_progress logger metadata observability page_initializer pagination path_resolver paths performance_collector performance_report profile progress retry rich_console sections swizzle text theme_registry theme_resolution thread_local traceback_config traceback_renderer url_normalization url_strategy Rendering Parsers ᗢ Caret Down Link Copy URL External Open LLM text Copy Copy LLM text Share with AI Ask Claude Ask ChatGPT Ask Gemini Ask Copilot Module rendering.parsers.mistune Mistune parser implementation - fast with full documentation features. View source 1 Class Classes MistuneParser Parser using mistune library. Faster with full documentation features. Supported features: - Table… 15 Caret Right Parser using mistune library. Faster with full documentation features. Supported features: Tables (GFM) Fenced code blocks Strikethrough Task lists Autolinks TOC generation (custom implementation) Admonitions (custom plugin) Footnotes (custom plugin) Definition lists (custom plugin) Variable substitution (custom plugin) - NEW! Inherits from BaseMarkdownParser Methods 9 Tag supports_ast property Check if this parser supports true AST output. Mistune natively supports AST o… bool Caret Right def supports_ast(self) -> bool Check if this parser supports true AST output. Mistune natively supports AST output via renderer=None. Returns bool — True - Mistune supports AST output parse Parse Markdown content into HTML. 2 str Caret Right def parse(self, content: str, metadata: dict[str, Any]) -> str Parse Markdown content into HTML. Parameters 2 content str Markdown content to parse metadata dict[str, Any] Page metadata (includes source path for validation warnings) Returns str — Rendered HTML string parse_with_toc Parse Markdown content and extract table of contents. Two-stage process: 1. Pa… 2 tuple[str, str] Caret Right def parse_with_toc(self, content: str, metadata: dict[str, Any]) -> tuple[str, str] Parse Markdown content and extract table of contents. Two-stage process: Parse markdown to HTML Inject heading anchors (IDs and headerlinks) Extract TOC from anchored headings Parameters 2 content str Markdown content to parse metadata dict[str, Any] Page metadata (includes source path for validation warnings) Returns tuple[str, str] — Tuple of (HTML with anchored headings, TOC HTML) parse_with_context Parse Markdown with variable substitution support. Variable Substitution: … 3 str Caret Right def parse_with_context(self, content: str, metadata: dict[str, Any], context: dict[str, Any]) -> str Parse Markdown with variable substitution support. Variable Substitution: Enables {{ page.title }}, {{ site.baseurl }}, etc. in markdown content. Uses a separate mistune instance (_md_with_vars) with preprocessing. Lazy Initialization: _md_with_vars is created on first use and cached thereafter. This happens once per parser instance (i.e., once per thread). Important: In parallel builds with max_workers=N: N parser instances created (main: self.md) N variable parser instances created (vars: self._md_with_vars) Total: 2N mistune instances, but only 1 of each per thread This is optimal - each thread uses its cached instances Parser Reuse: The parser with VariableSubstitutionPlugin is cached and reused. Only the context is updated per page (fast operation). This avoids expensive parser re-initialization (~10ms) for every page. Parameters 3 content str Markdown content to parse metadata dict[str, Any] Page metadata context dict[str, Any] Variable context (page, site, config) Returns str — Rendered HTML with variables substituted Performance: First call (per thread): Creates _md_with_vars (~10ms) Subsequent calls: Reuses cached parser (~0ms overhead) Variable preprocessing: ~0.5ms per page Markdown parsing: ~1-5ms per page parse_with_toc_and_context Parse Markdown with variable substitution and extract TOC. Single-pass parsing… 3 tuple[str, str] Caret Right def parse_with_toc_and_context(self, content: str, metadata: dict[str, Any], context: dict[str, Any]) -> tuple[str, str] Parse Markdown with variable substitution and extract TOC. Single-pass parsing with VariableSubstitutionPlugin for {{ vars }}. ARCHITECTURE DECISION: Separation of Concerns SUPPORTED in markdown content: {{ page.metadata.xxx }} - Variable substitution {{ site.config.xxx }} - Site configuration access Code blocks naturally stay literal (AST-level protection) NOT SUPPORTED in markdown content: {% if %} - Conditional blocks {% for %} - Loop constructs Complex Jinja2 logic WHY: These belong in TEMPLATES, not markdown content. Use conditionals and loops in your page templates: <!-- templates/page.html --> <article> {% if page.metadata.enterprise %} <div class="enterprise-badge">Enterprise</div> {% endif %} {{ content }} <!-- Markdown renders here --> </article> This design: Keeps parsing simple and fast (single pass) Separates content parsing from template logic Maintains performance (no preprocessing overhead) Makes code blocks work naturally Parameters 3 content str Markdown content to parse metadata dict[str, Any] Page metadata context dict[str, Any] Variable context (page, site, config) Returns tuple[str, str] — Tuple of (HTML with anchored headings, TOC HTML) enable_cross_references Enable cross-reference support with [[link]] syntax. Should be called after co… 1 None Caret Right def enable_cross_references(self, xref_index: dict[str, Any]) -> None Enable cross-reference support with [[link]] syntax. Should be called after content discovery when xref_index is built. Creates CrossReferencePlugin for post-processing HTML output. Also stores xref_index on the renderer for directive access (e.g., cards :pull:). Performance: O(1) - just stores reference to index Thread-safe: Each thread-local parser instance needs this called once Parameters 1 xref_index dict[str, Any] Pre-built cross-reference index from site discovery parse_to_ast Parse Markdown content to AST tokens. Uses Mistune's built-in AST support by p… 2 list[dict[str, Any]] Caret Right def parse_to_ast(self, content: str, metadata: dict[str, Any]) -> list[dict[str, Any]] Parse Markdown content to AST tokens. Uses Mistune's built-in AST support by parsing with renderer=None. The AST is a list of token dictionaries representing the document structure. Performance: Parsing cost is similar to parse() (same tokenization) AST is more memory-efficient than HTML for caching Multiple outputs can be generated from single AST Parameters 2 content str Raw Markdown content metadata dict[str, Any] Page metadata (unused, for interface compatibility) Returns list[dict[str, Any]] — List of AST token dictionaries render_ast Render AST tokens to HTML. Uses Mistune's renderer to convert AST tokens back … 1 str Caret Right def render_ast(self, ast: list[dict[str, Any]]) -> str Render AST tokens to HTML. Uses Mistune's renderer to convert AST tokens back to HTML. This enables parse-once, render-many patterns. Parameters 1 ast list[dict[str, Any]] List of AST token dictionaries from parse_to_ast() Returns str — Rendered HTML string parse_with_ast Parse content and return AST, HTML, and TOC together. Single-pass parsing that… 2 tuple[list[dict[str… Caret Right def parse_with_ast(self, content: str, metadata: dict[str, Any]) -> tuple[list[dict[str, Any]], str, str] Parse content and return AST, HTML, and TOC together. Single-pass parsing that returns all outputs efficiently. Use this when you need both AST (for caching) and HTML (for display). Parameters 2 content str Raw Markdown content metadata dict[str, Any] Page metadata Returns tuple[list[dict[str, Any]], str, str] — Tuple of (AST tokens, HTML content, TOC HTML) Performance: Single parse pass for AST Single render pass for HTML TOC extracted from HTML (fast regex) ~30% overhead vs parse() alone, but saves re-parsing Internal Methods 6 Caret Right __init__ Initialize the mistune parser with plugins. 1 None Caret Right def __init__(self, enable_highlighting: bool = True) -> None Initialize the mistune parser with plugins. Parameters 1 enable_highlighting bool Enable Pygments syntax highlighting for code blocks (defaults to True for backward compatibility) Parser Instances: This parser is typically created via thread-local caching. With parallel builds (max_workers=N), you'll see N instances created - one per worker thread. This is OPTIMAL, not a bug! Internal Structure: - self.md: Main mistune instance for standard parsing - self._md_with_vars: Created lazily for pages with {{ var }} syntax Both instances share plugins (cross-references, etc.) but have different preprocessing (variable substitution). _create_syntax_highlighting_plugin Create a Mistune plugin that adds Pygments syntax highlighting to code blocks. 0 Callable[[Any], None] Caret Right def _create_syntax_highlighting_plugin(self) -> Callable[[Any], None] Create a Mistune plugin that adds Pygments syntax highlighting to code blocks. Returns Callable[[Any], None] — Plugin function that modifies the renderer to add syntax highlighting _escape_jinja_blocks Escape raw Jinja2 block delimiters in HTML content. This converts "{%"/"%}" in… 1 str Caret Right def _escape_jinja_blocks(self, html: str) -> str Escape raw Jinja2 block delimiters in HTML content. This converts "{%"/"%}" into HTML entities so any documentation examples do not appear as unrendered template syntax in the final HTML. Parameters 1 html str Returns str _inject_heading_anchors Inject IDs into heading tags using fast regex (5-10x faster than BS4). Exclude… 1 str Caret Right def _inject_heading_anchors(self, html: str) -> str Inject IDs into heading tags using fast regex (5-10x faster than BS4). Excludes headings inside blockquotes from getting IDs (so they don't appear in TOC). Single-pass regex replacement handles: h2, h3, h4 headings (matching python-markdown's toc_depth) Existing IDs (preserves them) Heading content with nested HTML Generates clean slugs from heading text Skips headings inside <blockquote> tags Parameters 1 html str HTML content from markdown parser Returns str — HTML with heading IDs added (except those in blockquotes) _extract_toc Extract table of contents from HTML with anchored headings using fast regex (5-… 1 str Caret Right def _extract_toc(self, html: str) -> str Extract table of contents from HTML with anchored headings using fast regex (5-8x faster than BS4). Builds a nested list of links to heading anchors. Expects headings to have IDs (anchors handled by theme). Parameters 1 html str HTML content with heading IDs and headerlinks Returns str — TOC as HTML (div.toc > ul > li > a structure) _slugify Convert text to a URL-friendly slug. Matches python-markdown's default slugify … 1 str Caret Right def _slugify(self, text: str) -> str Convert text to a URL-friendly slug. Matches python-markdown's default slugify behavior. Uses bengal.utils.text.slugify with HTML unescaping enabled. Limits slug length to prevent overly long IDs from headers with code. Parameters 1 text str Text to slugify Returns str — Slugified text (max 100 characters) ← Previous factory Next → native_html List © 2025 Bengal ᓚᘏᗢ window.BENGAL_LAZY_ASSETS = { tabulator: '/bengal/assets/js/tabulator.min.js', dataTable: '/bengal/assets/js/data-table.js', mermaidToolbar: '/bengal/assets/js/mermaid-toolbar.9de5abba.js', mermaidTheme: '/bengal/assets/js/mermaid-theme.344822c5.js', graphMinimap: '/bengal/assets/js/graph-minimap.cc7e42e3.js', graphContextual: '/bengal/assets/js/graph-contextual.440e59c6.js' }; window.BENGAL_ICONS = { close: '/bengal/assets/icons/close.911d4fe1.svg', enlarge: '/bengal/assets/icons/enlarge.652035e5.svg', copy: '/bengal/assets/icons/copy.3d56e945.svg', 'download-svg': '/bengal/assets/icons/download.04f07e1b.svg', 'download-png': '/bengal/assets/icons/image.c34dfd40.svg', 'zoom-in': '/bengal/assets/icons/zoom-in.237b4a83.svg', 'zoom-out': '/bengal/assets/icons/zoom-out.38857c77.svg', reset: '/bengal/assets/icons/reset.d26dba29.svg' }; Arrow Up X -------------------------------------------------------------------------------- Metadata: - Author: lbliii - Word Count: 2392 - Reading Time: 12 minutes