# _protocol URL: /api/_protocol/ Section: api -------------------------------------------------------------------------------- _protocol - Rosettes window.BENGAL_THEME_DEFAULTS = { appearance: 'light', palette: 'brown-bengal' }; window.Bengal = window.Bengal || {}; window.Bengal.enhanceBaseUrl = '/rosettes/assets/js/enhancements'; window.Bengal.watchDom = true; window.Bengal.debug = false; window.Bengal.enhanceUrls = { 'toc': '/rosettes/assets/js/enhancements/toc.632a9783.js', 'docs-nav': '/rosettes/assets/js/enhancements/docs-nav.57e4b129.js', 'tabs': '/rosettes/assets/js/enhancements/tabs.aac9e817.js', 'lightbox': '/rosettes/assets/js/enhancements/lightbox.1ca22aa1.js', 'interactive': '/rosettes/assets/js/enhancements/interactive.fc077855.js', 'mobile-nav': '/rosettes/assets/js/enhancements/mobile-nav.d991657f.js', 'action-bar': '/rosettes/assets/js/enhancements/action-bar.d62417f4.js', 'copy-link': '/rosettes/assets/js/enhancements/copy-link.7d9a5c29.js', 'data-table': '/rosettes/assets/js/enhancements/data-table.1f5bc1eb.js', 'lazy-loaders': '/rosettes/assets/js/enhancements/lazy-loaders.a5c38245.js', 'holo': '/rosettes/assets/js/enhancements/holo.ee13c841.js', 'link-previews': '/rosettes/assets/js/enhancements/link-previews.8d906535.js' }; (function () { try { var defaults = window.BENGAL_THEME_DEFAULTS || { appearance: 'system', palette: '' }; var defaultAppearance = defaults.appearance; if (defaultAppearance === 'system') { defaultAppearance = (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? 'dark' : 'light'; } var storedTheme = localStorage.getItem('bengal-theme'); var storedPalette = localStorage.getItem('bengal-palette'); var theme = storedTheme ? (storedTheme === 'system' ? defaultAppearance : storedTheme) : defaultAppearance; var palette = storedPalette ?? defaults.palette; document.documentElement.setAttribute('data-theme', theme); if (palette) { document.documentElement.setAttribute('data-palette', palette); } } catch (e) { document.documentElement.setAttribute('data-theme', 'light'); } })(); { "prerender": [ { "where": { "and": [ { "href_matches": "/docs/*" }, { "not": { "selector_matches": "[data-external], [target=_blank], .external" } } ] }, "eagerness": "conservative" } ], "prefetch": [ { "where": { "and": [ { "href_matches": "/*" }, { "not": { "selector_matches": "[data-external], [target=_blank], .external" } } ] }, "eagerness": "conservative" } ] } Skip to main content Magnifying Glass ESC Recent Clear Magnifying Glass No results for "" Start typing to search... ↑↓ Navigate ↵ Open ESC Close Powered by Lunr ⌾⌾⌾ DocumentationInfoAboutArrow ClockwiseGet StartedCodeHighlightingPaletteStylingStarburstExtendingFormattersNoteTutorialsBookmarkReferenceReleasesDevGitHubAPI Reference Magnifying Glass Search ⌘K Palette Appearance Chevron Down Mode Monitor System Sun Light Moon Dark Palette Snow Lynx Brown Bengal Silver Bengal Charcoal Bengal Blue Bengal List ⌾⌾⌾ Magnifying Glass Search X Close Documentation Caret Down Info About Arrow Clockwise Get Started Code Highlighting Palette Styling Starburst Extending Formatters Note Tutorials Bookmark Reference Releases Dev Caret Down GitHub API Reference Palette Appearance Chevron Down Mode Monitor System Sun Light Moon Dark Palette Snow Lynx Brown Bengal Silver Bengal Charcoal Bengal Blue Bengal Rosettes API Reference Caret Right Formatters html null terminal Caret Right Lexers _scanners _state_machine bash_sm c_sm clojure_sm cpp_sm css_sm csv_sm cuda_sm cue_sm dart_sm diff_sm dockerfile_sm elixir_sm gleam_sm go_sm graphql_sm groovy_sm haskell_sm hcl_sm html_sm ini_sm java_sm javascript_sm jinja_sm json_sm julia_sm kida_sm kotlin_sm lua_sm makefile_sm markdown_sm mojo_sm nginx_sm nim_sm perl_sm php_sm pkl_sm plaintext_sm powershell_sm protobuf_sm python_sm r_sm ruby_sm rust_sm scala_sm sql_sm stan_sm swift_sm toml_sm tree_sm triton_sm typescript_sm v_sm xml_sm yaml_sm zig_sm Caret Right Themes _mapping _palette _roles palettes _config _escape _formatter_registry _parallel _protocol _registry _types delegate rosettes Rosettes API Reference ᗢ Caret Down Link Copy URL External Open LLM text Copy Copy LLM text Share with AI Ask Claude Ask ChatGPT Ask Gemini Ask Copilot Module _protocol Protocol definitions for Rosettes. Defines the contracts that lexers and formatters must implement. All implementations must be thread-safe. Design Philosophy: Rosettes uses Protocol (structural typing) rather than abstract base classes. This enables: Duck typing with safety: Any object with the right methods works No inheritance hierarchy: Cleaner, more flexible implementations Easy testing: Create minimal mock implementations without inheritance Gradual adoption: Existing classes can satisfy protocols without changes When to Implement: Lexer Protocol: ✅ Adding support for a new programming language ✅ Creating a specialized lexer (e.g., for a DSL) ✅ Wrapping an external tokenizer with Rosettes interface Formatter Protocol: ✅ New output format (LaTeX, RTF, Markdown, etc.) ✅ Custom HTML structure for specific frameworks ✅ Integration with external rendering systems Example (Custom Lexer): class MyDslLexer: name = "mydsl" aliases = ("dsl",) filenames = ("*.dsl",) mimetypes = () def tokenize(self, code, config=None, start=0, end=None): # Your tokenization logic here yield Token(TokenType.TEXT, code, 1, 1) def tokenize_fast(self, code, start=0, end=None): yield (TokenType.TEXT, code) Example (Custom Formatter): class MarkdownFormatter: name = "markdown" def format(self, tokens, config=None): for token in tokens: if token.type == TokenType.KEYWORD: yield f"**{token.value}**" else: yield token.value See Also: rosettes.lexers._state_machine: Base class for hand-written lexers rosettes.formatters.html: Reference Formatter implementation rosettes._registry: How lexers are registered and looked up 2Classes Classes Lexer 6 ▼ Protocol for tokenizers. Implementations must be thread-safe — no mutable shared state. The tokeni… Protocol for tokenizers. Implementations must be thread-safe — no mutable shared state. The tokenize method should only use local variables. Thread-Safety Contract: tokenize() must use only local variables No instance state mutation during tokenization Class-level constants (KEYWORDS, etc.) must be immutable (frozenset) Performance Contract: O(n) time complexity guaranteed (no backtracking) Single pass through input (no lookahead beyond current position) Streaming output (yield tokens as found) Methods name 0 str ▼ The canonical name of this lexer (e.g., 'python'). property def name(self) -> str Returns str aliases 0 tuple[str, ...] ▼ Alternative names for this lexer (e.g., ('py', 'python3')). property def aliases(self) -> tuple[str, ...] Returns tuple[str, ...] filenames 0 tuple[str, ...] ▼ Glob patterns for files this lexer handles (e.g., ('*.py',)). property def filenames(self) -> tuple[str, ...] Returns tuple[str, ...] mimetypes 0 tuple[str, ...] ▼ MIME types this lexer handles. property def mimetypes(self) -> tuple[str, ...] Returns tuple[str, ...] tokenize 4 Iterator[Token] ▼ Tokenize source code into a stream of tokens. def tokenize(self, code: str, config: LexerConfig | None = None, start: int = 0, end: int | None = None) -> Iterator[Token] Parameters Name Type Description code — The source code to tokenize. config — Optional lexer configuration. Default: None start — Starting index in the source string. Default: 0 end — Optional ending index in the source string. Default: None Returns Iterator[Token] tokenize_fast 3 Iterator[tuple[TokenType… ▼ Fast tokenization without position tracking. Yields minimal (type, value) tupl… def tokenize_fast(self, code: str, start: int = 0, end: int | None = None) -> Iterator[tuple[TokenType, str]] Fast tokenization without position tracking. Yields minimal (type, value) tuples for maximum speed. Use when line/column info is not needed. Parameters Name Type Description code — The source code to tokenize. start — Starting index in the source string. Default: 0 end — Optional ending index in the source string. Default: None Returns Iterator[tuple[TokenType, str]] Formatter 5 ▼ Protocol for output formatters. Implementations must be thread-safe — use only local variables in … Protocol for output formatters. Implementations must be thread-safe — use only local variables in format(). Formatter instances should be immutable (frozen dataclasses recommended). Thread-Safety Contract: format() must use only local variables Instance state must be immutable after construction No side effects (file I/O, network, etc.) Streaming Contract: format() yields chunks as they're ready (generator) format_string() convenience method joins chunks Callers can start processing before full output is ready Fast Path: format_fast() accepts (TokenType, value) tuples instead of Token objects Avoids Token construction overhead when position info not needed ~20% faster for simple highlighting without line numbers Methods name 0 str ▼ The canonical name of this formatter (e.g., 'html'). property def name(self) -> str Returns str format 2 Iterator[str] ▼ Format tokens into output chunks. def format(self, tokens: Iterator[Token], config: FormatConfig | None = None) -> Iterator[str] Parameters Name Type Description tokens — Stream of tokens to format. config — Optional formatter configuration. Default: None Returns Iterator[str] format_fast 2 Iterator[str] ▼ Fast formatting without position tracking. def format_fast(self, tokens: Iterator[tuple[TokenType, str]], config: FormatConfig | None = None) -> Iterator[str] Parameters Name Type Description tokens — Stream of (TokenType, value) tuples. config — Optional formatter configuration. Default: None Returns Iterator[str] format_string 2 str ▼ Format tokens and return as a single string. Convenience method that joins for… def format_string(self, tokens: Iterator[Token], config: FormatConfig | None = None) -> str Format tokens and return as a single string. Convenience method that joins format() output. Parameters Name Type Description tokens — Stream of tokens to format. config — Optional formatter configuration. Default: None Returns str Complete formatted string. format_string_fast 2 str ▼ Fast format and return as a single string. Convenience method that joins forma… def format_string_fast(self, tokens: Iterator[tuple[TokenType, str]], config: FormatConfig | None = None) -> str Fast format and return as a single string. Convenience method that joins format_fast() output. Parameters Name Type Description tokens — Stream of (TokenType, value) tuples. config — Optional formatter configuration. Default: None Returns str Complete formatted string. ← Previous _parallel Next → _registry List © 2026 Rosettes built in ᓚᘏᗢ { "linkPreviews": { "enabled": true, "hoverDelay": 200, "hideDelay": 150, "showSection": true, "showReadingTime": true, "showWordCount": true, "showDate": true, "showTags": true, "maxTags": 3, "includeSelectors": [".prose"], "excludeSelectors": ["nav", ".toc", ".breadcrumb", ".pagination", ".card", "[class*='-card']", ".tab-nav", "[class*='-widget']", ".child-items", ".content-tiles"], "allowedHosts": [], "allowedSchemes": ["https"], "hostFailureThreshold": 3 } } window.BENGAL_LAZY_ASSETS = { tabulator: '/rosettes/assets/js/tabulator.min.js', dataTable: '/rosettes/assets/js/data-table.js', mermaidToolbar: '/rosettes/assets/js/mermaid-toolbar.9de5abba.js', mermaidTheme: '/rosettes/assets/js/mermaid-theme.344822c5.js', graphMinimap: '/rosettes/assets/js/graph-minimap.ff04e939.js', graphContextual: '/rosettes/assets/js/graph-contextual.355458ba.js' }; window.BENGAL_ICONS = { close: '/rosettes/assets/icons/close.911d4fe1.svg', enlarge: '/rosettes/assets/icons/enlarge.652035e5.svg', copy: '/rosettes/assets/icons/copy.3d56e945.svg', 'download-svg': '/rosettes/assets/icons/download.04f07e1b.svg', 'download-png': '/rosettes/assets/icons/image.c34dfd40.svg', 'zoom-in': '/rosettes/assets/icons/zoom-in.237b4a83.svg', 'zoom-out': '/rosettes/assets/icons/zoom-out.38857c77.svg', reset: '/rosettes/assets/icons/reset.d26dba29.svg' }; Arrow Up -------------------------------------------------------------------------------- Metadata: - Word Count: 1395 - Reading Time: 7 minutes