# tokens URL: /api/tokens/ Section: api -------------------------------------------------------------------------------- tokens - Patitas window.BENGAL_THEME_DEFAULTS = { appearance: 'light', palette: 'brown-bengal' }; window.Bengal = window.Bengal || {}; window.Bengal.enhanceBaseUrl = '/patitas/assets/js/enhancements'; window.Bengal.watchDom = true; window.Bengal.debug = false; window.Bengal.enhanceUrls = { 'toc': '/patitas/assets/js/enhancements/toc.632a9783.js', 'docs-nav': '/patitas/assets/js/enhancements/docs-nav.57e4b129.js', 'tabs': '/patitas/assets/js/enhancements/tabs.aac9e817.js', 'lightbox': '/patitas/assets/js/enhancements/lightbox.1ca22aa1.js', 'interactive': '/patitas/assets/js/enhancements/interactive.fc077855.js', 'mobile-nav': '/patitas/assets/js/enhancements/mobile-nav.d991657f.js', 'action-bar': '/patitas/assets/js/enhancements/action-bar.d62417f4.js', 'copy-link': '/patitas/assets/js/enhancements/copy-link.7d9a5c29.js', 'data-table': '/patitas/assets/js/enhancements/data-table.1f5bc1eb.js', 'lazy-loaders': '/patitas/assets/js/enhancements/lazy-loaders.a5c38245.js', 'holo': '/patitas/assets/js/enhancements/holo.ee13c841.js', 'link-previews': '/patitas/assets/js/enhancements/link-previews.8d906535.js' }; (function () { try { var defaults = window.BENGAL_THEME_DEFAULTS || { appearance: 'system', palette: '' }; var defaultAppearance = defaults.appearance; if (defaultAppearance === 'system') { defaultAppearance = (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? 'dark' : 'light'; } var storedTheme = localStorage.getItem('bengal-theme'); var storedPalette = localStorage.getItem('bengal-palette'); var theme = storedTheme ? (storedTheme === 'system' ? defaultAppearance : storedTheme) : defaultAppearance; var palette = storedPalette ?? defaults.palette; document.documentElement.setAttribute('data-theme', theme); if (palette) { document.documentElement.setAttribute('data-palette', palette); } } catch (e) { document.documentElement.setAttribute('data-theme', 'light'); } })(); { "prerender": [ { "where": { "and": [ { "href_matches": "/docs/*" }, { "not": { "selector_matches": "[data-external], [target=_blank], .external" } } ] }, "eagerness": "conservative" } ], "prefetch": [ { "where": { "and": [ { "href_matches": "/*" }, { "not": { "selector_matches": "[data-external], [target=_blank], .external" } } ] }, "eagerness": "conservative" } ] } Skip to main content Magnifying Glass ESC Recent Clear Magnifying Glass No results for "" Start typing to search... ↑↓ Navigate ↵ Open ESC Close Powered by Lunr ฅᨐฅ DocumentationArrow ClockwiseGet StartedCodeSyntaxDirectivesStarburstExtendingBookmarkReferenceInfoAboutTroubleshootingReleasesDevGitHubPatitas API Reference Magnifying Glass Search ⌘K Palette Appearance Chevron Down Mode Monitor System Sun Light Moon Dark Palette Snow Lynx Brown Bengal Silver Bengal Charcoal Bengal Blue Bengal List ฅᨐฅ Magnifying Glass Search X Close Documentation Caret Down Arrow Clockwise Get Started Code Syntax Directives Starburst Extending Bookmark Reference Info About Troubleshooting Releases Dev Caret Down GitHub Patitas API Reference Palette Appearance Chevron Down Mode Monitor System Sun Light Moon Dark Palette Snow Lynx Brown Bengal Silver Bengal Charcoal Bengal Blue Bengal Patitas API Reference Caret Right Directives Caret Right Builtins admonition container dropdown tabs contracts decorator options protocol registry Caret Right Lexer Caret Right Classifiers directive fence footnote heading html link_ref list quote thematic Caret Right Scanners block directive fence html core modes Caret Right Parsing Caret Right Blocks Caret Right List blank_line indent item_blocks marker mixin nested trace types core directive footnote table Caret Right Inline core emphasis links match_registry special tokens charsets containers token_nav Caret Right Plugins autolinks footnotes math strikethrough table task_lists Caret Right Renderers html Caret Right Roles Caret Right Builtins formatting icons math reference protocol registry Caret Right Utils hashing logger text errors highlighting icons location nodes parser patitas protocols stringbuilder tokens Patitas API Reference ᗢ Caret Down Link Copy URL External Open LLM text Copy Copy LLM text Share with AI Ask Claude Ask ChatGPT Ask Gemini Ask Copilot Module tokens Token and TokenType definitions for the Patitas lexer. The lexer produces a stream of Token objects that the parser consumes. Each Token has a type, value, and source location. Thread Safety: Token is frozen (immutable) and safe to share across threads. TokenType is an enum (inherently immutable). 2Classes Classes TokenType 0 ▼ Token types produced by the lexer. Organized by category for clarity: - Document structure (EOF, B… Token types produced by the lexer. Organized by category for clarity: Document structure (EOF, BLANK_LINE) Block elements (headings, code blocks, quotes, lists) Inline elements (text, emphasis, links, images) Directives and roles Token 7 ▼ A token produced by the lexer. Tokens are the atomic units passed from lexer to parser. Each token… A token produced by the lexer. Tokens are the atomic units passed from lexer to parser. Each token has a type, string value, and source location. Thread Safety: Frozen dataclass ensures immutability for safe sharing. Attributes Name Type Description type TokenType The token type (from TokenType enum) value str The raw string value from source location SourceLocation Source location for error messages line_indent int Pre-computed indent level of the line (spaces, tabs expand to 4). Set by lexer at token creation; -1 if not computed. Methods lineno 0 int ▼ Line number (convenience accessor). property def lineno(self) -> int Returns int col 0 int ▼ Column offset (convenience accessor). property def col(self) -> int Returns int Internal Methods 1 ▼ __repr__ 0 str ▼ Compact repr for debugging. def __repr__(self) -> str Returns str ← Previous stringbuilder List © 2026 Patitas built in ᓚᘏᗢ { "linkPreviews": { "enabled": true, "hoverDelay": 200, "hideDelay": 150, "showSection": true, "showReadingTime": true, "showWordCount": true, "showDate": true, "showTags": true, "maxTags": 3, "includeSelectors": [".prose"], "excludeSelectors": ["nav", ".toc", ".breadcrumb", ".pagination", ".card", "[class*='-card']", ".tab-nav", "[class*='-widget']", ".child-items", ".content-tiles"], "allowedHosts": [], "allowedSchemes": ["https"], "hostFailureThreshold": 3 } } window.BENGAL_LAZY_ASSETS = { tabulator: '/patitas/assets/js/tabulator.min.js', dataTable: '/patitas/assets/js/data-table.js', mermaidToolbar: '/patitas/assets/js/mermaid-toolbar.9de5abba.js', mermaidTheme: '/patitas/assets/js/mermaid-theme.344822c5.js', graphMinimap: '/patitas/assets/js/graph-minimap.ff04e939.js', graphContextual: '/patitas/assets/js/graph-contextual.355458ba.js' }; window.BENGAL_ICONS = { close: '/patitas/assets/icons/close.911d4fe1.svg', enlarge: '/patitas/assets/icons/enlarge.652035e5.svg', copy: '/patitas/assets/icons/copy.3d56e945.svg', 'download-svg': '/patitas/assets/icons/download.04f07e1b.svg', 'download-png': '/patitas/assets/icons/image.c34dfd40.svg', 'zoom-in': '/patitas/assets/icons/zoom-in.237b4a83.svg', 'zoom-out': '/patitas/assets/icons/zoom-out.38857c77.svg', reset: '/patitas/assets/icons/reset.d26dba29.svg' }; Arrow Up -------------------------------------------------------------------------------- Metadata: - Word Count: 791 - Reading Time: 4 minutes