# _parallel URL: /api/_parallel/ Section: api -------------------------------------------------------------------------------- _parallel - Rosettes window.BENGAL_THEME_DEFAULTS = { appearance: 'light', palette: 'brown-bengal' }; window.Bengal = window.Bengal || {}; window.Bengal.enhanceBaseUrl = '/rosettes/assets/js/enhancements'; window.Bengal.watchDom = true; window.Bengal.debug = false; window.Bengal.enhanceUrls = { 'toc': '/rosettes/assets/js/enhancements/toc.632a9783.js', 'docs-nav': '/rosettes/assets/js/enhancements/docs-nav.57e4b129.js', 'tabs': '/rosettes/assets/js/enhancements/tabs.aac9e817.js', 'lightbox': '/rosettes/assets/js/enhancements/lightbox.1ca22aa1.js', 'interactive': '/rosettes/assets/js/enhancements/interactive.fc077855.js', 'mobile-nav': '/rosettes/assets/js/enhancements/mobile-nav.d991657f.js', 'action-bar': '/rosettes/assets/js/enhancements/action-bar.d62417f4.js', 'copy-link': '/rosettes/assets/js/enhancements/copy-link.7d9a5c29.js', 'data-table': '/rosettes/assets/js/enhancements/data-table.1f5bc1eb.js', 'lazy-loaders': '/rosettes/assets/js/enhancements/lazy-loaders.a5c38245.js', 'holo': '/rosettes/assets/js/enhancements/holo.ee13c841.js', 'link-previews': '/rosettes/assets/js/enhancements/link-previews.8d906535.js' }; (function () { try { var defaults = window.BENGAL_THEME_DEFAULTS || { appearance: 'system', palette: '' }; var defaultAppearance = defaults.appearance; if (defaultAppearance === 'system') { defaultAppearance = (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? 'dark' : 'light'; } var storedTheme = localStorage.getItem('bengal-theme'); var storedPalette = localStorage.getItem('bengal-palette'); var theme = storedTheme ? (storedTheme === 'system' ? defaultAppearance : storedTheme) : defaultAppearance; var palette = storedPalette ?? defaults.palette; document.documentElement.setAttribute('data-theme', theme); if (palette) { document.documentElement.setAttribute('data-palette', palette); } } catch (e) { document.documentElement.setAttribute('data-theme', 'light'); } })(); { "prerender": [ { "where": { "and": [ { "href_matches": "/docs/*" }, { "not": { "selector_matches": "[data-external], [target=_blank], .external" } } ] }, "eagerness": "conservative" } ], "prefetch": [ { "where": { "and": [ { "href_matches": "/*" }, { "not": { "selector_matches": "[data-external], [target=_blank], .external" } } ] }, "eagerness": "conservative" } ] } Skip to main content Magnifying Glass ESC Recent Clear Magnifying Glass No results for "" Start typing to search... ↑↓ Navigate ↵ Open ESC Close Powered by Lunr ⌾⌾⌾ DocumentationInfoAboutArrow ClockwiseGet StartedCodeHighlightingPaletteStylingStarburstExtendingFormattersNoteTutorialsBookmarkReferenceReleasesDevGitHubAPI Reference Magnifying Glass Search ⌘K Palette Appearance Chevron Down Mode Monitor System Sun Light Moon Dark Palette Snow Lynx Brown Bengal Silver Bengal Charcoal Bengal Blue Bengal List ⌾⌾⌾ Magnifying Glass Search X Close Documentation Caret Down Info About Arrow Clockwise Get Started Code Highlighting Palette Styling Starburst Extending Formatters Note Tutorials Bookmark Reference Releases Dev Caret Down GitHub API Reference Palette Appearance Chevron Down Mode Monitor System Sun Light Moon Dark Palette Snow Lynx Brown Bengal Silver Bengal Charcoal Bengal Blue Bengal Rosettes API Reference Caret Right Formatters html null terminal Caret Right Lexers _scanners _state_machine bash_sm c_sm clojure_sm cpp_sm css_sm csv_sm cuda_sm cue_sm dart_sm diff_sm dockerfile_sm elixir_sm gleam_sm go_sm graphql_sm groovy_sm haskell_sm hcl_sm html_sm ini_sm java_sm javascript_sm jinja_sm json_sm julia_sm kida_sm kotlin_sm lua_sm makefile_sm markdown_sm mojo_sm nginx_sm nim_sm perl_sm php_sm pkl_sm plaintext_sm powershell_sm protobuf_sm python_sm r_sm ruby_sm rust_sm scala_sm sql_sm stan_sm swift_sm toml_sm tree_sm triton_sm typescript_sm v_sm xml_sm yaml_sm zig_sm Caret Right Themes _mapping _palette _roles palettes _config _escape _formatter_registry _parallel _protocol _registry _types delegate rosettes Rosettes API Reference ᗢ Caret Down Link Copy URL External Open LLM text Copy Copy LLM text Share with AI Ask Claude Ask ChatGPT Ask Gemini Ask Copilot Module _parallel Parallel tokenization for free-threaded Python (3.14t+). Enables true parallel tokenization of large files by splitting at safe boundaries and processing chunks concurrently. Design Philosophy: This module exists for one purpose: maximum throughput on Python 3.14t. On GIL Python (3.13 and earlier), threads cannot truly parallelize CPU-bound work. But on free-threaded Python 3.14t, Rosettes lexers use only local variables, enabling true parallel tokenization. Architecture: Split Detection: Find safe split points (newlines) to avoid cutting tokens in half Chunking: Divide code into ~64KB chunks with position metadata Parallel Execution: Tokenize chunks using ThreadPoolExecutor Line Adjustment: Fix line numbers for chunks after the first Ordered Merge: Yield tokens in original source order When to Use: ✅ Large files (>128KB) on Python 3.14t ✅ Batch processing many files with highlight_many() ❌ Small files (< 128KB) — sequential is faster (thread overhead) ❌ GIL Python — no parallelism benefit Performance: Sequential: ~50µs per 100-line file Parallel (4 workers, 3.14t): ~15µs per file for batches of 100+ The crossover point is ~8 items or ~128KB of code. Thread-Safety: Safe by design: Lexers use only local variables Chunks are independent (no shared state) Token lists are created per-chunk, then merged Limitations: Splitting at newlines may not be safe for all languages (e.g., heredocs spanning lines). This is rare in practice. Memory: Holds all chunk results before yielding See Also: rosettes.highlight_many: High-level parallel API rosettes.tokenize_many: Parallel tokenization without formatting 1Class4Functions Classes _Chunk 3 ▼ A chunk of source code with position metadata. A chunk of source code with position metadata. Attributes Name Type Description text str — start_offset int — start_line int — Functions is_free_threaded 0 bool ▼ Check if running on free-threaded Python (3.14t+). def is_free_threaded() -> bool Returns bool _find_safe_splits 2 list[int] ▼ Find safe split points (newlines) for parallel tokenization. We split at newli… def _find_safe_splits(code: str, target_chunk_size: int) -> list[int] Find safe split points (newlines) for parallel tokenization. We split at newlines to avoid splitting in the middle of tokens. This is a heuristic that works for most languages. Parameters Name Type Description code str Source code to split. target_chunk_size int Target size for each chunk. Returns list[int] _make_chunks 2 list[_Chunk] ▼ Split code into chunks at the given positions. def _make_chunks(code: str, splits: list[int]) -> list[_Chunk] Parameters Name Type Description code str Source code to split. splits list[int] List of positions to split at. Returns list[_Chunk] tokenize_parallel 2 Iterator[Token] ▼ Parallel tokenization for large files. Only beneficial on free-threaded Python… def tokenize_parallel(lexer: StateMachineLexer, code: str) -> Iterator[Token] Parallel tokenization for large files. Only beneficial on free-threaded Python (3.14t+). Falls back to sequential on GIL Python. Parameters Name Type Description lexer StateMachineLexer The lexer to use. code str Source code to tokenize. Returns Iterator[Token] ← Previous _formatter_registry Next → _protocol List © 2026 Rosettes built in ᓚᘏᗢ { "linkPreviews": { "enabled": true, "hoverDelay": 200, "hideDelay": 150, "showSection": true, "showReadingTime": true, "showWordCount": true, "showDate": true, "showTags": true, "maxTags": 3, "includeSelectors": [".prose"], "excludeSelectors": ["nav", ".toc", ".breadcrumb", ".pagination", ".card", "[class*='-card']", ".tab-nav", "[class*='-widget']", ".child-items", ".content-tiles"], "allowedHosts": [], "allowedSchemes": ["https"], "hostFailureThreshold": 3 } } window.BENGAL_LAZY_ASSETS = { tabulator: '/rosettes/assets/js/tabulator.min.js', dataTable: '/rosettes/assets/js/data-table.js', mermaidToolbar: '/rosettes/assets/js/mermaid-toolbar.9de5abba.js', mermaidTheme: '/rosettes/assets/js/mermaid-theme.344822c5.js', graphMinimap: '/rosettes/assets/js/graph-minimap.ff04e939.js', graphContextual: '/rosettes/assets/js/graph-contextual.355458ba.js' }; window.BENGAL_ICONS = { close: '/rosettes/assets/icons/close.911d4fe1.svg', enlarge: '/rosettes/assets/icons/enlarge.652035e5.svg', copy: '/rosettes/assets/icons/copy.3d56e945.svg', 'download-svg': '/rosettes/assets/icons/download.04f07e1b.svg', 'download-png': '/rosettes/assets/icons/image.c34dfd40.svg', 'zoom-in': '/rosettes/assets/icons/zoom-in.237b4a83.svg', 'zoom-out': '/rosettes/assets/icons/zoom-out.38857c77.svg', reset: '/rosettes/assets/icons/reset.d26dba29.svg' }; Arrow Up -------------------------------------------------------------------------------- Metadata: - Word Count: 960 - Reading Time: 5 minutes