# _types URL: /api/_types/ Section: api -------------------------------------------------------------------------------- _types - Rosettes window.BENGAL_THEME_DEFAULTS = { appearance: 'light', palette: 'brown-bengal' }; window.Bengal = window.Bengal || {}; window.Bengal.enhanceBaseUrl = '/rosettes/assets/js/enhancements'; window.Bengal.watchDom = true; window.Bengal.debug = false; window.Bengal.enhanceUrls = { 'toc': '/rosettes/assets/js/enhancements/toc.632a9783.js', 'docs-nav': '/rosettes/assets/js/enhancements/docs-nav.57e4b129.js', 'tabs': '/rosettes/assets/js/enhancements/tabs.aac9e817.js', 'lightbox': '/rosettes/assets/js/enhancements/lightbox.1ca22aa1.js', 'interactive': '/rosettes/assets/js/enhancements/interactive.fc077855.js', 'mobile-nav': '/rosettes/assets/js/enhancements/mobile-nav.d991657f.js', 'action-bar': '/rosettes/assets/js/enhancements/action-bar.d62417f4.js', 'copy-link': '/rosettes/assets/js/enhancements/copy-link.7d9a5c29.js', 'data-table': '/rosettes/assets/js/enhancements/data-table.1f5bc1eb.js', 'lazy-loaders': '/rosettes/assets/js/enhancements/lazy-loaders.a5c38245.js', 'holo': '/rosettes/assets/js/enhancements/holo.ee13c841.js', 'link-previews': '/rosettes/assets/js/enhancements/link-previews.8d906535.js' }; (function () { try { var defaults = window.BENGAL_THEME_DEFAULTS || { appearance: 'system', palette: '' }; var defaultAppearance = defaults.appearance; if (defaultAppearance === 'system') { defaultAppearance = (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? 'dark' : 'light'; } var storedTheme = localStorage.getItem('bengal-theme'); var storedPalette = localStorage.getItem('bengal-palette'); var theme = storedTheme ? (storedTheme === 'system' ? defaultAppearance : storedTheme) : defaultAppearance; var palette = storedPalette ?? defaults.palette; document.documentElement.setAttribute('data-theme', theme); if (palette) { document.documentElement.setAttribute('data-palette', palette); } } catch (e) { document.documentElement.setAttribute('data-theme', 'light'); } })(); { "prerender": [ { "where": { "and": [ { "href_matches": "/docs/*" }, { "not": { "selector_matches": "[data-external], [target=_blank], .external" } } ] }, "eagerness": "conservative" } ], "prefetch": [ { "where": { "and": [ { "href_matches": "/*" }, { "not": { "selector_matches": "[data-external], [target=_blank], .external" } } ] }, "eagerness": "conservative" } ] } Skip to main content Magnifying Glass ESC Recent Clear Magnifying Glass No results for "" Start typing to search... ↑↓ Navigate ↵ Open ESC Close Powered by Lunr ⌾⌾⌾ DocumentationInfoAboutArrow ClockwiseGet StartedCodeHighlightingPaletteStylingStarburstExtendingFormattersNoteTutorialsBookmarkReferenceReleasesDevGitHubAPI Reference Magnifying Glass Search ⌘K Palette Appearance Chevron Down Mode Monitor System Sun Light Moon Dark Palette Snow Lynx Brown Bengal Silver Bengal Charcoal Bengal Blue Bengal List ⌾⌾⌾ Magnifying Glass Search X Close Documentation Caret Down Info About Arrow Clockwise Get Started Code Highlighting Palette Styling Starburst Extending Formatters Note Tutorials Bookmark Reference Releases Dev Caret Down GitHub API Reference Palette Appearance Chevron Down Mode Monitor System Sun Light Moon Dark Palette Snow Lynx Brown Bengal Silver Bengal Charcoal Bengal Blue Bengal Rosettes API Reference Caret Right Formatters html null terminal Caret Right Lexers _scanners _state_machine bash_sm c_sm clojure_sm cpp_sm css_sm csv_sm cuda_sm cue_sm dart_sm diff_sm dockerfile_sm elixir_sm gleam_sm go_sm graphql_sm groovy_sm haskell_sm hcl_sm html_sm ini_sm java_sm javascript_sm jinja_sm json_sm julia_sm kida_sm kotlin_sm lua_sm makefile_sm markdown_sm mojo_sm nginx_sm nim_sm perl_sm php_sm pkl_sm plaintext_sm powershell_sm protobuf_sm python_sm r_sm ruby_sm rust_sm scala_sm sql_sm stan_sm swift_sm toml_sm tree_sm triton_sm typescript_sm v_sm xml_sm yaml_sm zig_sm Caret Right Themes _mapping _palette _roles palettes _config _escape _formatter_registry _parallel _protocol _registry _types delegate rosettes Rosettes API Reference ᗢ Caret Down Link Copy URL External Open LLM text Copy Copy LLM text Share with AI Ask Claude Ask ChatGPT Ask Gemini Ask Copilot Module _types Core types for Rosettes syntax highlighting. Thread-safe, immutable types for tokenization. Design Philosophy: Types in this module are designed for maximum performance and safety: 1. **Immutable**: Token is a NamedTuple, TokenType is a StrEnum 2. **Minimal memory**: Token is ~64 bytes (vs ~200 for a regular object) 3. **Hashable**: Tokens can be used in sets/dicts for deduplication 4. **Thread-safe**: Immutability means no synchronization needed What Goes in TokenType: ✅ DO include: Language keywords (KEYWORD, KEYWORD_DECLARATION, etc.) Operators and punctuation (OPERATOR, PUNCTUATION) Literals (STRING, NUMBER, etc.) Comments (COMMENT, COMMENT_MULTILINE, etc.) Names (NAME, NAME_FUNCTION, NAME_CLASS, etc.) ❌ DON'T include: Formatting hints (indentation level, line breaks) Editor-specific tokens (folding markers, etc.) Language-specific tokens (use generic categories) Pygments Compatibility: TokenType values are the CSS class suffixes used by Pygments themes. This means existing Pygments stylesheets work with Rosettes output: TokenType.KEYWORD = "k" → def TokenType.NAME_FUNCTION = "nf" → my_func Use css_class_style="pygments" in highlight() for this compatibility. Use css_class_style="semantic" for readable classes like .syntax-function. See Also: rosettes.themes._roles: Higher-level semantic roles for theming rosettes.themes._mapping: TokenType → SyntaxRole mapping rosettes.formatters.html: How TokenTypes become CSS classes 2Classes Classes TokenType 0 ▼ Semantic token types with Pygments-compatible CSS class names. Each value is the CSS class suffix … Semantic token types with Pygments-compatible CSS class names. Each value is the CSS class suffix used by Pygments themes. This ensures drop-in compatibility with existing Pygments stylesheets. Categories: Keywords: KEYWORD, KEYWORD_CONSTANT, KEYWORD_DECLARATION, etc. Names: NAME, NAME_FUNCTION, NAME_CLASS, NAME_BUILTIN, etc. Literals: STRING, NUMBER, NUMBER_FLOAT, etc. Operators: OPERATOR, OPERATOR_WORD Punctuation: PUNCTUATION, PUNCTUATION_MARKER Comments: COMMENT, COMMENT_SINGLE, COMMENT_MULTILINE, etc. Generic: TEXT, WHITESPACE, ERROR (for diffs, errors, etc.) Usage: >>> from rosettes import TokenType >>> TokenType.KEYWORD <TokenType.KEYWORD: 'k'> >>> TokenType.KEYWORD.value # CSS class suffix 'k' Token 4 ▼ Immutable token — thread-safe, minimal memory. A Token represents a single lexical unit from sourc… Immutable token — thread-safe, minimal memory. A Token represents a single lexical unit from source code. Tokens are immutable NamedTuples for thread-safety and memory efficiency. Memory: Each Token uses ~64 bytes (NamedTuple overhead + references). A typical 100-line Python file produces ~500 tokens (~32KB). Thread-Safety: Tokens are immutable and can be safely shared across threads. No defensive copying needed when passing tokens between workers. Fast Path: When position info is not needed, use tokenize_fast() which yields (TokenType, str) tuples instead of Token objects for ~20% speedup. Attributes Name Type Description type TokenType The semantic type of the token (e.g., TokenType.KEYWORD). value str The actual text content of the token (e.g., "def"). line int 1-based line number where token starts. column int 1-based column number where token starts. ← Previous _registry Next → delegate List © 2026 Rosettes built in ᓚᘏᗢ { "linkPreviews": { "enabled": true, "hoverDelay": 200, "hideDelay": 150, "showSection": true, "showReadingTime": true, "showWordCount": true, "showDate": true, "showTags": true, "maxTags": 3, "includeSelectors": [".prose"], "excludeSelectors": ["nav", ".toc", ".breadcrumb", ".pagination", ".card", "[class*='-card']", ".tab-nav", "[class*='-widget']", ".child-items", ".content-tiles"], "allowedHosts": [], "allowedSchemes": ["https"], "hostFailureThreshold": 3 } } window.BENGAL_LAZY_ASSETS = { tabulator: '/rosettes/assets/js/tabulator.min.js', dataTable: '/rosettes/assets/js/data-table.js', mermaidToolbar: '/rosettes/assets/js/mermaid-toolbar.9de5abba.js', mermaidTheme: '/rosettes/assets/js/mermaid-theme.344822c5.js', graphMinimap: '/rosettes/assets/js/graph-minimap.ff04e939.js', graphContextual: '/rosettes/assets/js/graph-contextual.355458ba.js' }; window.BENGAL_ICONS = { close: '/rosettes/assets/icons/close.911d4fe1.svg', enlarge: '/rosettes/assets/icons/enlarge.652035e5.svg', copy: '/rosettes/assets/icons/copy.3d56e945.svg', 'download-svg': '/rosettes/assets/icons/download.04f07e1b.svg', 'download-png': '/rosettes/assets/icons/image.c34dfd40.svg', 'zoom-in': '/rosettes/assets/icons/zoom-in.237b4a83.svg', 'zoom-out': '/rosettes/assets/icons/zoom-out.38857c77.svg', reset: '/rosettes/assets/icons/reset.d26dba29.svg' }; Arrow Up -------------------------------------------------------------------------------- Metadata: - Word Count: 934 - Reading Time: 5 minutes