89:"/_next/static/immutable/chunks/30sab0f73xdfw.js" 8a:"/_next/static/immutable/chunks/36x3jrwbe3drn.js" 8b:I[328986,["$2","$3","$4","$5","$6","$24","$25","$26","$27","$28","$29","$2a","$2b","$2c","$2d","$2e","$2f","$30","$31","$32","$33","$34","$35","$36","$37","$38","$39","$3a","$3b","$3c","$3d","$3e","$3f","$40","$41","$42","$43","$44","$45","$89","$8a"],"FullTraceClient",1] 8c:Tbb8,Command: /bin/zsh -lc 'rg --files' Chunk ID: 21a0fb Wall time: 0.0000 seconds Process exited with code 0 Original token count: 716 Output: src/analysis.ts src/layout.test.ts src/layout.ts src/measurement.ts src/test-data.ts src/line-break.ts src/text-modules.d.ts src/bidi.ts research-data/system-ui-size-scan.json corpora/th-nithan-vetal-story-7.txt corpora/th-nithan-vetal-story-1.txt corpora/ja-rashomon.txt corpora/hi-eidgah.txt corpora/ja-kumo-no-ito.txt corpora/dashboard.json corpora/README.md corpora/zh-zhufu.txt corpora/ur-chughd.txt corpora/chrome-sampled.json corpora/mixed-app-text.txt corpora/zh-guxiang.txt corpora/en-gatsby-opening.txt corpora/he-masaot-binyamin-metudela.txt corpora/STATUS.md corpora/representative.json corpora/my-bad-deeds-return-to-you-teacher.txt corpora/chrome-step10.json corpora/TAXONOMY.md corpora/ko-unsu-joh-eun-nal.txt corpora/km-prachum-reuang-preng-khmer-volume-7-stories-1-10.txt corpora/sources.json corpora/my-cunning-heron-teacher.txt corpora/ar-risalat-al-ghufran-part-1.txt corpora/ar-al-bukhala.txt pages/demos/bubbles.html pages/demos/variable-typographic-ascii.ts pages/demos/justification-comparison.model.ts pages/demos/justification-comparison.ts pages/demos/editorial-engine.ts pages/demos/dynamic-layout-text.ts pages/demos/justification-comparison.ui.ts pages/demos/bubbles.ts pages/demos/justification-comparison.data.ts pages/demos/wrap-geometry.ts pages/demos/variable-typographic-ascii.html pages/demos/svg.d.ts pages/demos/editorial-engine.html pages/demos/dynamic-layout.html pages/demos/rich-note.html pages/demos/masonry/index.ts pages/demos/masonry/shower-thoughts.json pages/demos/masonry/index.html pages/demos/accordion.ts pages/demos/accordion.html pages/demos/bubbles-shared.ts pages/demos/justification-comparison.html pages/demos/rich-note.ts pages/demos/index.html pages/demos/dynamic-layout.ts pages/assets/claude-symbol.svg pages/assets/openai-symbol.svg pages/corpus.html pages/benchmark.ts pages/diagnostic-utils.ts pages/benchmark.html pages/emoji-test.html pages/gatsby.ts pages/probe.html pages/probe.ts pages/report-utils.ts pages/accuracy.html pages/justification-comparison.html pages/accuracy.ts pages/corpus.ts pages/gatsby.html DEVELOPMENT.md AGENTS.md oxlintrc.json tsconfig.json scripts/corpus-check.ts scripts/probe-check.ts scripts/gatsby-sweep.ts scripts/report-server.ts scripts/corpus-font-matrix.ts scripts/build-demo-site.ts scripts/pre-wrap-check.ts scripts/status-dashboard.ts scripts/accuracy-check.ts scripts/gatsby-check.ts scripts/corpus-taxonomy.ts scripts/corpus-sweep.ts scripts/corpus-representative.ts scripts/corpus-status.ts scripts/package-smoke-test.ts scripts/benchmark-check.ts scripts/browser-automation.ts benchmarks/safari.json benchmarks/chrome.json tsconfig.build.json package.json thoughts.md RESEARCH.md README.md accuracy/firefox.json accuracy/safari.json accuracy/chrome.json TODO.md status/dashboard.json bun.lock shared/navigation-state.ts STATUS.md CHANGELOG.md LICENSE 8d:T1241,Command: /bin/zsh -lc "sed -n '1,220p' package.json" Chunk ID: 9aed31 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1129 Output: { "name": "@chenglou/pretext", "version": "0.0.3", "license": "MIT", "type": "module", "publishConfig": { "access": "public" }, "main": "./dist/layout.js", "types": "./dist/layout.d.ts", "exports": { ".": { "types": "./dist/layout.d.ts", "import": "./dist/layout.js", "default": "./dist/layout.js" }, "./demos/*": "./pages/demos/*", "./assets/*": "./pages/assets/*", "./package.json": "./package.json" }, "files": [ "CHANGELOG.md", "dist", "src", "!src/layout.test.ts", "!src/test-data.ts", "pages/demos", "pages/assets", "LICENSE" ], "scripts": { "accuracy-check": "bun run scripts/accuracy-check.ts", "accuracy-check:firefox": "ACCURACY_CHECK_BROWSER=firefox bun run scripts/accuracy-check.ts", "accuracy-check:safari": "ACCURACY_CHECK_BROWSER=safari bun run scripts/accuracy-check.ts", "accuracy-snapshot": "bun run scripts/accuracy-check.ts --full --output=accuracy/chrome.json", "accuracy-snapshot:firefox": "ACCURACY_CHECK_BROWSER=firefox bun run scripts/accuracy-check.ts --full --output=accuracy/firefox.json", "accuracy-snapshot:safari": "ACCURACY_CHECK_BROWSER=safari bun run scripts/accuracy-check.ts --full --output=accuracy/safari.json", "benchmark-check": "bun run scripts/benchmark-check.ts", "benchmark-check:safari": "BENCHMARK_CHECK_BROWSER=safari bun run scripts/benchmark-check.ts", "build:package": "rm -rf dist && tsc -p tsconfig.build.json", "check": "tsc && oxlint --type-aware src", "corpus-check": "bun run scripts/corpus-check.ts", "corpus-check:safari": "CORPUS_CHECK_BROWSER=safari bun run scripts/corpus-check.ts", "corpus-font-matrix": "bun run scripts/corpus-font-matrix.ts", "corpus-font-matrix:safari": "CORPUS_CHECK_BROWSER=safari bun run scripts/corpus-font-matrix.ts", "corpus-representative": "bun run scripts/corpus-representative.ts --output=corpora/representative.json", "corpus-status": "bun run scripts/corpus-status.ts", "corpus-status:refresh": "bun run scripts/corpus-representative.ts --output=corpora/representative.json && bun run scripts/corpus-sweep.ts --all --samples=9 --output=corpora/chrome-sampled.json && bun run scripts/corpus-sweep.ts --all --start=300 --end=900 --step=10 --output=corpora/chrome-step10.json && bun run scripts/corpus-status.ts", "corpus-sweep": "bun run scripts/corpus-sweep.ts", "corpus-sweep:safari": "CORPUS_CHECK_BROWSER=safari bun run scripts/corpus-sweep.ts", "corpus-taxonomy": "bun run scripts/corpus-taxonomy.ts", "gatsby-check": "bun run scripts/gatsby-check.ts", "gatsby-check:safari": "GATSBY_CHECK_BROWSER=safari bun run scripts/gatsby-check.ts", "gatsby-sweep": "bun run scripts/gatsby-sweep.ts", "gatsby-sweep:safari": "GATSBY_CHECK_BROWSER=safari bun run scripts/gatsby-sweep.ts", "package-smoke-test": "bun run scripts/package-smoke-test.ts", "prepack": "rm -rf dist && tsc -p tsconfig.build.json", "pre-wrap-check": "bun run scripts/pre-wrap-check.ts", "probe-check": "bun run scripts/probe-check.ts", "probe-check:safari": "PROBE_CHECK_BROWSER=safari bun run scripts/probe-check.ts", "status-dashboard": "bun run scripts/status-dashboard.ts", "site:build": "rm -rf site && bun run scripts/build-demo-site.ts", "start": "HOST=${HOST:-127.0.0.1}; PORT=3000; pids=$(lsof -tiTCP:$PORT -sTCP:LISTEN 2>/dev/null); if [ -n \"$pids\" ]; then echo \"Freeing port $PORT: terminating $pids\"; kill $pids 2>/dev/null || true; sleep 1; pids=$(lsof -tiTCP:$PORT -sTCP:LISTEN 2>/dev/null); if [ -n \"$pids\" ]; then echo \"Port $PORT still busy: killing $pids\"; kill -9 $pids 2>/dev/null || true; fi; fi; bun pages/*.html pages/demos/*.html pages/demos/*/index.html --host=$HOST:$PORT", "start:lan": "HOST=0.0.0.0 bun run start", "start:watch": "HOST=${HOST:-127.0.0.1}; PORT=3000; pids=$(lsof -tiTCP:$PORT -sTCP:LISTEN 2>/dev/null); if [ -n \"$pids\" ]; then echo \"Freeing port $PORT: terminating $pids\"; kill $pids 2>/dev/null || true; sleep 1; pids=$(lsof -tiTCP:$PORT -sTCP:LISTEN 2>/dev/null); if [ -n \"$pids\" ]; then echo \"Port $PORT still busy: killing $pids\"; kill -9 $pids 2>/dev/null || true; fi; fi; bun pages/*.html pages/demos/*.html pages/demos/*/index.html --watch --no-clear-screen --host=$HOST:$PORT" }, "devDependencies": { "@types/bun": "latest", "oxlint": "^1.51.0", "oxlint-tsgolint": "^0.15.0", "tsgolint": "^0.0.1", "typescript": "6.0.2" } } 8e:T2391,Command: /bin/zsh -lc "sed -n '1,260p' README.md" Chunk ID: 9f95ef Wall time: 0.0000 seconds Process exited with code 0 Original token count: 2238 Output: # Pretext Pure JavaScript/TypeScript library for multiline text measurement & layout. Fast, accurate & supports all the languages you didn't even know about. Allows rendering to DOM, Canvas, SVG and soon, server-side. Pretext side-steps the need for DOM measurements (e.g. `getBoundingClientRect`, `offsetHeight`), which trigger layout reflow, one of the most expensive operations in the browser. It implements its own text measurement logic, using the browsers' own font engine as ground truth (very AI-friendly iteration method). ## Installation ```sh npm install @chenglou/pretext ``` ## Demos Clone the repo, run `bun install`, then `bun start`, and open the `/demos` in your browser (no trailing slash. Bun devserver bugs on those) Alternatively, see them live at [chenglou.me/pretext](https://chenglou.me/pretext/). Some more at [somnai-dreams.github.io/pretext-demos](https://somnai-dreams.github.io/pretext-demos/) ## API Pretext serves 2 use cases: ### 1. Measure a paragraph's height _without ever touching DOM_ ```ts import { prepare, layout } from '@chenglou/pretext' const prepared = prepare('AGI 春天到了. بدأت الرحلة 🚀', '16px Inter') const { height, lineCount } = layout(prepared, textWidth, 20) // pure arithmetics. No DOM layout & reflow! ``` `prepare()` does the one-time work: normalize whitespace, segment the text, apply glue rules, measure the segments with canvas, and return an opaque handle. `layout()` is the cheap hot path after that: pure arithmetic over cached widths. Do not rerun `prepare()` for the same text and configs; that'd defeat its precomputation. For example, on resize, only rerun `layout()`. If you want textarea-like text where ordinary spaces, `\t` tabs, and `\n` hard breaks stay visible, pass `{ whiteSpace: 'pre-wrap' }` to `prepare()`: ```ts const prepared = prepare(textareaValue, '16px Inter', { whiteSpace: 'pre-wrap' }) const { height } = layout(prepared, textareaWidth, 20) ``` On the current checked-in benchmark snapshot: - `prepare()` is about `19ms` for the shared 500-text batch - `layout()` is about `0.09ms` for that same batch We support all the languages you can imagine, including emojis and mixed-bidi, and caters to specific browser quirks The returned height is the crucial last piece for unlocking web UI's: - proper virtualization/occlusion without guesstimates & caching - fancy userland layouts: masonry, JS-driven flexbox-like implementations, nudging a few layout values without CSS hacks (imagine that), etc. - _development time_ verification (especially now with AI) that labels on e.g. buttons don't overflow to the next line, browser-free - prevent layout shift when new text loads and you wanna re-anchor the scroll position ### 2. Lay out the paragraph lines manually yourself Switch out `prepare` with `prepareWithSegments`, then: - `layoutWithLines()` gives you all the lines at a fixed width: ```ts import { prepareWithSegments, layoutWithLines } from '@chenglou/pretext' const prepared = prepareWithSegments('AGI 春天到了. بدأت الرحلة 🚀', '18px "Helvetica Neue"') const { lines } = layoutWithLines(prepared, 320, 26) // 320px max width, 26px line height for (let i = 0; i < lines.length; i++) ctx.fillText(lines[i].text, 0, i * 26) ``` - `walkLineRanges()` gives you line widths and cursors without building the text strings: ```ts let maxW = 0 walkLineRanges(prepared, 320, line => { if (line.width > maxW) maxW = line.width }) // maxW is now the widest line — the tightest container width that still fits the text! This multiline "shrink wrap" has been missing from web ``` - `layoutNextLine()` lets you route text one row at a time when width changes as you go: ```ts let cursor = { segmentIndex: 0, graphemeIndex: 0 } let y = 0 // Flow text around a floated image: lines beside the image are narrower while (true) { const width = y < image.bottom ? columnWidth - image.width : columnWidth const line = layoutNextLine(prepared, cursor, width) if (line === null) break ctx.fillText(line.text, 0, y) cursor = line.end y += 26 } ``` This usage allows rendering to canvas, SVG, WebGL and (eventually) server-side. ### API Glossary Use-case 1 APIs: ```ts prepare(text: string, font: string, options?: { whiteSpace?: 'normal' | 'pre-wrap' }): PreparedText // one-time text analysis + measurement pass, returns an opaque value to pass to `layout()`. Make sure `font` is synced with your css `font` declaration shorthand (e.g. size, weight, style, family) for the text you're measuring. `font` is the same format as what you'd use for `myCanvasContext.font = ...`, e.g. `16px Inter`. layout(prepared: PreparedText, maxWidth: number, lineHeight: number): { height: number, lineCount: number } // calculates text height given a max width and lineHeight. Make sure `lineHeight` is synced with your css `line-height` declaration for the text you're measuring. ``` Use-case 2 APIs: ```ts prepareWithSegments(text: string, font: string, options?: { whiteSpace?: 'normal' | 'pre-wrap' }): PreparedTextWithSegments // same as `prepare()`, but returns a richer structure for manual line layouts needs layoutWithLines(prepared: PreparedTextWithSegments, maxWidth: number, lineHeight: number): { height: number, lineCount: number, lines: LayoutLine[] } // high-level api for manual layout needs. Accepts a fixed max width for all lines. Similar to `layout()`'s return, but additionally returns the lines info walkLineRanges(prepared: PreparedTextWithSegments, maxWidth: number, onLine: (line: LayoutLineRange) => void): number // low-level api for manual layout needs. Accepts a fixed max width for all lines. Calls `onLine` once per line with its actual calculated line width and start/end cursors, without building line text strings. Very useful for certain cases where you wanna speculatively test a few width and height boundaries (e.g. binary search a nice width value by repeatedly calling walkLineRanges and checking the line count, and therefore height, is "nice" too. You can have text messages shrinkwrap and balanced text layout this way). After walkLineRanges calls, you'd call layoutWithLines once, with your satisfying max width, to get the actual lines info. layoutNextLine(prepared: PreparedTextWithSegments, start: LayoutCursor, maxWidth: number): LayoutLine | null // iterator-like api for laying out each line with a different width! Returns the LayoutLine starting from `start`, or `null` when the paragraph's exhausted. Pass the previous line's `end` cursor as the next `start`. type LayoutLine = { text: string // Full text content of this line, e.g. 'hello world' width: number // Measured width of this line, e.g. 87.5 start: LayoutCursor // Inclusive start cursor in prepared segments/graphemes end: LayoutCursor // Exclusive end cursor in prepared segments/graphemes } type LayoutLineRange = { width: number // Measured width of this line, e.g. 87.5 start: LayoutCursor // Inclusive start cursor in prepared segments/graphemes end: LayoutCursor // Exclusive end cursor in prepared segments/graphemes } type LayoutCursor = { segmentIndex: number // Segment index in prepareWithSegments' prepared rich segment stream graphemeIndex: number // Grapheme index within that segment; `0` at segment boundaries } ``` Other helpers: ```ts clearCache(): void // clears Pretext's shared internal caches used by prepare() and prepareWithSegments(). Useful if your app cycles through many different fonts or text variants and you want to release the accumulated cache setLocale(locale?: string): void // optional (by default we use the current locale). Sets locale for future prepare() and prepareWithSegments(). Internally, it also calls clearCache(). Setting a new locale doesn't affect existing prepare() and prepareWithSegments() states (no mutations to them) ``` ## Caveats Pretext doesn't try to be a full font rendering engine (yet?). It currently targets the common text setup: - `white-space: normal` - `word-break: normal` - `overflow-wrap: break-word` - `line-break: auto` - If you pass `{ whiteSpace: 'pre-wrap' }`, ordinary spaces, `\t` tabs, and `\n` hard breaks are preserved instead of collapsed. Tabs follow the default browser-style `tab-size: 8`. The other wrapping defaults stay the same: `word-break: normal`, `overflow-wrap: break-word`, and `line-break: auto`. - `system-ui` is unsafe for `layout()` accuracy on macOS. Use a named font. - Because the default target includes `overflow-wrap: break-word`, very narrow widths can still break inside words, but only at grapheme boundaries. ## Develop See [DEVELOPMENT.md](DEVELOPMENT.md) for the dev setup and commands. ## Credits Sebastian Markbage first planted the seed with [text-layout](https://github.com/chenglou/text-layout) last decade. His design — canvas `measureText` for shaping, bidi from pdf.js, streaming line breaking — informed the architecture we kept pushing forward here. 8f:T2bb7,Command: /bin/zsh -lc "sed -n '1,260p' RESEARCH.md" Chunk ID: deb083 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 2759 Output: # Research Log Everything we tried, measured, and learned while building this library. For the current compact browser-accuracy / benchmark snapshot, see `STATUS.md`. For the current compact corpus / sweep snapshot, see `corpora/STATUS.md`. For the shared mismatch vocabulary, see `corpora/TAXONOMY.md`. ## Current steering summary This log is historical. The current practical steering picture is: - Japanese has two real canaries (`羅生門`, `蜘蛛の糸`), both clean at anchor widths and both still exposing a small positive one-line field on broader Chrome sweeps. - Chinese has two long-form canaries (`祝福`, `故鄉`) showing the same broad Chrome-positive / Safari-clean split, with real font sensitivity between `Songti SC` and `PingFang SC`. - Myanmar still has two real canaries with residual Chrome/Safari disagreement around quote/follower-style classes, so it remains the main unresolved Southeast Asian frontier. - Urdu has a real Nastaliq/Naskh canary (`چغد`) with the same narrow-width negative field in Chrome and Safari, so it is clearly a shaping/context class rather than dirty data or a browser-only quirk. It remains parked rather than actively tuned. - Arabic coarse corpora are clean; the remaining work there is mostly a fine-width edge-fit class, not the old preprocessing/corpus-hygiene problems. - Mixed app text still matters because it catches product-shaped classes that books miss, especially soft-hyphen and extractor-sensitive cases. ## The problem: DOM measurement interleaving When UI components independently measure text heights with DOM reads like `getBoundingClientRect()`, each read can force synchronous layout. If those reads interleave with writes, the browser can end up relaying out the whole document repeatedly. The goal here was always the same: - do the expensive text work once in `prepare()` - keep `layout()` arithmetic-only - make resize-driven relayout cheap and coordination-free ## Approach 1: Canvas measureText + word-width caching Canvas `measureText()` avoids DOM layout. It goes straight to the browser's font engine. That led to the basic two-phase model: - `prepare(text, font)` — segment text, measure segments, cache widths - `layout(prepared, maxWidth, lineHeight)` — walk cached widths with pure arithmetic That architecture held up. The broad browser sweeps are now clean in Chrome, Safari, and Firefox, and the hot `layout()` path is still the core product win. ## Rejected: DOM-based or string-reconstruction measurement in the hot path Several alternatives were tried and rejected: - measuring full candidate lines as strings during `layout()` - moving measurement into hidden DOM elements during `prepare()` - using SVG `getComputedTextLength()` The pattern was consistent: - they either reintroduced DOM reads - or they were slower than the current two-phase model - or they looked cleaner locally but regressed the actual benchmark path The important keep was architectural, not algorithmic: - `layout()` stayed arithmetic-only on cached widths ## Discovery: system-ui font resolution mismatch Canvas and DOM resolve `system-ui` to different font variants on macOS at certain sizes: Machine-readable scan: - [research-data/system-ui-size-scan.json](/Users/chenglou/github/pretext/research-data/system-ui-size-scan.json) In the recorded scan, mismatches clustered at `10-12px`, `14px`, and `26px`. `13px`, `15-25px`, and `27-28px` were exact. macOS uses SF Pro Text at smaller sizes and SF Pro Display at larger sizes. Canvas and DOM switch between them at different thresholds. Practical conclusion: - use a named font if accuracy matters - keep `system-ui` documented as unsafe - if we ever support it properly, the believable path is a narrow prepare-time DOM fallback for detected bad tuples What did **not** look trustworthy enough: - lookup tables - naive scaling - guessed resolved-font substitution ## Discovery: word-by-word sum accuracy Canvas is internally consistent enough that summing measured segments works very well, but not perfectly. Over a full paragraph, tiny adjacency differences can accumulate into a line-edge error. The keeps were small and semantic: - merge punctuation into the preceding word before measuring - let trailing collapsible spaces hang instead of forcing a break What did **not** survive: - full-string verification in `layout()` - uniform rescaling - generic pair-level correction models The broad lesson was that local semantic preprocessing paid off more than clever runtime correction. ## Discovery: text-shaper is a useful reference, not a runtime replacement `text-shaper` was useful reference material, especially for Unicode coverage and bidi ideas, but not a replacement for the current browser-facing model. What was worth taking: - broader Unicode coverage, e.g. missing CJK extension blocks What was not worth taking: - its segmentation as a runtime replacement for `Intl.Segmenter` - its paragraph breaker as a substitute for browser-parity layout Bottom line: - good reference material - wrong runtime center of gravity for this repo ## Discovery: preserving ordinary spaces, hard breaks, and numeric tab stops is viable The smallest honest second whitespace mode turned out to be: - preserve ordinary spaces - preserve `\n` hard breaks - preserve tabs with default browser-style tab stops - leave the other wrapping defaults alone That became: - `{ whiteSpace: 'pre-wrap' }` What mattered: - preserved spaces still hang at line end - consecutive hard breaks keep empty lines - a trailing final hard break does **not** invent an extra empty line - tabs advance to the next default browser tab stop from the current line start The mode now covers the textarea-like cases we cared about, and the broad browser sweeps plus the dedicated `pre-wrap` oracle are green. One important tooling lesson also came out of this: - keep a small permanent oracle suite - justify it once with a broader brute-force validation pass - do not keep the brute-force pass forever once it has done its job ## Discovery: emoji canvas/DOM width discrepancy Chrome and Firefox on macOS can measure emoji wider in canvas than in DOM at small sizes. Safari does not share the same discrepancy. What held up: - detect the discrepancy by comparing canvas emoji width against actual DOM emoji width per font - cache that correction - keep it outside the hot layout path This is now one of the small browser-profile shims that is actually justified. ## Retired HarfBuzz probe path We briefly kept a headless HarfBuzz backend in the repo for server-side measurement probes. What it taught us: - it was useful for research and algorithm probes - it was not close enough to our active browser-grounded path to justify keeping it in the main repo - isolated Arabic words in that probe path needed explicit LTR direction to avoid misleading widths So if HarfBuzz comes up again later, treat it as explored territory: - useful as a research reference - not the runtime direction for Pretext - not a substitute for browser-oracle or browser-canvas validation ## Final browser sweep closure The last browser mismatches were not fixed by moving more work into `layout()`. That regressed the hot path and was reverted. What actually held up: - better preprocessing in `prepare()` - better browser diagnostics pages and scripts - a tiny browser-specific line-fit tolerance What did **not** change: - `layout()` stayed arithmetic-only That remains the right center of gravity for the project. ## Arabic frontier Arabic took several passes, but the pattern is clearer now. What survived: - merge no-space Arabic punctuation clusters during `prepare()` - e.g. `فيقول:وعليك`, `همزةٌ،ما` - treat Arabic punctuation-plus-mark clusters like `،ٍ` as left-sticky too - split `" " + combining marks` into plain space plus marks attached to the following word - use normalized slices and the exact corpus font during probe work - trust the better RTL diagnostics path instead of reconstructing offsets from rendered line text - clean obvious corpus/source artifacts instead of inventing new engine rules for them - allow a tiny non-Safari line-fit tolerance bump for the remaining positive fine-width field What did **not** survive: - pair correction models at segment boundaries - larger Arabic run-slice width models - broad phrase-level heuristics derived from one good-looking probe Those failed for the same reason in different sizes: - pair corrections were too local to move the real misses - run-slice widths were much heavier and still did not move the hard widths enough - both made `prepare()` or `layout()` materially worse without buying a clean Arabic field So the useful guardrail is: - if an Arabic idea starts by adding more shaping-aware width caches inside the current segment-sum architecture, be skeptical early - the Arabic keeps so far have been preprocessing, corpus cleanup, diagnostics, and tiny tolerance shims, not richer width-cache models Current read: - Arabic coarse corpora are healthy - the remaining work is much narrower now - the unresolved class looks like a mix of fine-width edge-fit and shaping/context, not another obvious preprocessing hole ## Long-form corpus canaries Once the main browser sweep became a regression gate, the long-form corpora became the real steering canaries. ### Mixed app text This is the most product-shaped canary. What it has been good for: - URL/query-string handling - escaped quote clusters - numeric expressions like `२४×७` - time ranges like `7:00-9:00` - emoji ZWJ runs - manual soft hyphens Important keep: - model URL/query strings as narrow structured units, not one giant breakable blob Current status: - almost entirely clean - one remaining extractor-sensitive soft-hyphen miss around `710px` still looks paragraph-scale or accumulation-sensitive rather than like a neat local bug ### Thai Thai exposed a product-shaped ASCII quote issue more than a dictionary-segmentation failure. The keep: - contextual ASCII quote glue during preprocessing Result: - two Thai prose corpora are healthy at anchor widths - sampled sweeps stayed clean enough that Thai now looks broader than one lucky story ### Khmer Khmer broadened the Southeast Asian class without immediately demanding new engine work. The keep: - preserve explicit zero-width separators from the source text Result: - anchors and sampled sweeps were clean enough to keep Khmer as a real canary ### Lao (rejected) The Lao corpus attempt was a source problem, not an engine problem. The raw text was wrapped print/legal text, which made it a dirty `white-space: normal` canary. We rejected it instead of normalizing nonsense into the repo. ### Myanmar Myanmar is still the main unresolved Southeast Asian frontier. What survived: - treat `၊` / `။` / `၍` / `၌` / `၏` as left-sticky during preprocessing - treat `၏` as medial glue in clusters like `ကျွန်ုပ်၏လက်မ` 90:T91b,Command: /bin/zsh -lc "sed -n '1,260p' thoughts.md" Chunk ID: 059ab7 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 544 Output: ## Personal Thoughts, In No Particular Order - My personal desire is to see end-developers being empowered to make all the layouts that makes UI more expressive, without sacrificing maintainability & performance, and without having to wait for permissions from the web committee. The current UI stereotypes across all platforms has been either: - a landing-page with few, floating text chunks, powered by GL - a blog article with mostly just text and no possible interactivity - a SaaS dashboard - a mobile UI with 2-3 rectangles' worth of UI - If you dig deep enough, 80% of CSS spec could be avoided if userland had better control over text. The paradigm of web layout shoves our text into a single-direction black hole, and to crawl those text metrics back incurs huge maintenance and performance overhead (ask AI about this). - The convenience angle of CSS is gradually being eroded by the fact that: - the more CSS expressivity we bake in, the worse the CSS perf becomes (against all wishes from everyone on the committee and userland), and "programming" in CSS (as opposed to just "declare things" in CSS) is something very few desire - AI alleviates the need of having more hard-coded CSS configs, which are becoming more dictionary-like rather compositional. - It's very hard to have new competing web browser implementations, because the specs are gigantic, and many engines (grassroot efforts, gamers-driven rewrites, languages-driven rewrites, etc.) eagerly chase & premature perf improvements before realizing that the specs, often written decades ago and disregarding modern perf & feature needs, throw a wrench in those architectures. As a first approximation, UI performance & developer ergonomics _cannot_ possibly have an order of magnitude improvement, because the bottleneck is in the specs themselves. The only way to circumvent this is to bring more capabilities to userland, in a hope to stop the spec from over-growing even more in the future. Stopping spec complexity is something every browser vendor can agree on (and ironically, sometime for completely opposing reasons). - The cost of any verifiable software will trend toward 0 91:T2ad6,Command: /bin/zsh -lc 'rg -n "''^(export |function |const |type |interface )" src/layout.ts src/analysis.ts src/line-break.ts src/measurement.ts src/bidi.ts' Chunk ID: c6a6f8 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 2676 Output: src/layout.ts:72:function getSharedGraphemeSegmenter(): Intl.Segmenter { src/layout.ts:83:type PreparedCore = { src/layout.ts:99:export type PreparedText = { src/layout.ts:103:type InternalPreparedText = PreparedText & PreparedCore src/layout.ts:107:export type PreparedTextWithSegments = InternalPreparedText & { src/layout.ts:111:export type LayoutCursor = { src/layout.ts:116:export type LayoutResult = { src/layout.ts:121:export type LayoutLine = { src/layout.ts:128:export type LayoutLineRange = { src/layout.ts:134:export type LayoutLinesResult = LayoutResult & { src/layout.ts:138:export type PrepareProfile = { src/layout.ts:147:export type PrepareOptions = { src/layout.ts:151:export type PreparedLineChunk = { src/layout.ts:159:function createEmptyPrepared(includeSegments: boolean): InternalPreparedText | PreparedTextWithSegments { src/layout.ts:191:function measureAnalysis( src/layout.ts:394:function mapAnalysisChunksToPreparedChunks( src/layout.ts:424:function prepareInternal( src/layout.ts:436:export function profilePrepare(text: string, font: string, options?: PrepareOptions): PrepareProfile { src/layout.ts:472:export function prepare(text: string, font: string, options?: PrepareOptions): PreparedText { src/layout.ts:478:export function prepareWithSegments(text: string, font: string, options?: PrepareOptions): PreparedTextWithSegments { src/layout.ts:482:function getInternalPrepared(prepared: PreparedText): InternalPreparedText { src/layout.ts:495:export function layout(prepared: PreparedText, maxWidth: number, lineHeight: number): LayoutResult { src/layout.ts:503:function getSegmentGraphemes( src/layout.ts:520:function getLineTextCache(prepared: PreparedTextWithSegments): Map { src/layout.ts:529:function lineHasDiscretionaryHyphen( src/layout.ts:542:function buildLineTextFromRange( src/layout.ts:581:function createLayoutLine( src/layout.ts:612:function materializeLayoutLine( src/layout.ts:628:function toLayoutLineRange(line: InternalLayoutLine): LayoutLineRange { src/layout.ts:642:function stepLineRange( src/layout.ts:652:function materializeLine( src/layout.ts:669:export function walkLineRanges( src/layout.ts:681:export function layoutNextLine( src/layout.ts:695:export function layoutWithLines(prepared: PreparedTextWithSegments, maxWidth: number, lineHeight: number): LayoutLinesResult { src/layout.ts:707:export function clearCache(): void { src/layout.ts:714:export function setLocale(locale?: string): void { src/line-break.ts:4:export type LineBreakCursor = { src/line-break.ts:9:export type PreparedLineBreakData = { src/line-break.ts:26:export type InternalLayoutLine = { src/line-break.ts:34:function canBreakAfter(kind: SegmentBreakKind): boolean { src/line-break.ts:44:function isSimpleCollapsibleSpace(kind: SegmentBreakKind): boolean { src/line-break.ts:48:function getTabAdvance(lineWidth: number, tabStopAdvance: number): number { src/line-break.ts:56:function getBreakableAdvance( src/line-break.ts:68:function fitSoftHyphenBreak( src/line-break.ts:94:function findChunkIndexForStart(prepared: PreparedLineBreakData, segmentIndex: number): number { src/line-break.ts:102:export function normalizeLineStart( src/line-break.ts:133:export function countPreparedLines(prepared: PreparedLineBreakData, maxWidth: number): number { src/line-break.ts:140:function countPreparedLinesSimple(prepared: PreparedLineBreakData, maxWidth: number): number { src/line-break.ts:204:function walkPreparedLinesSimple( src/line-break.ts:375:export function walkPreparedLines( src/line-break.ts:673:export function layoutNextLineRange( src/line-break.ts:937:function layoutNextLineRangeSimple( src/measurement.ts:3:export type SegmentMetrics = { src/measurement.ts:11:export type EngineProfile = { src/measurement.ts:19:const segmentMetricCaches = new Map>() src/measurement.ts:22:const emojiPresentationRe = /\p{Emoji_Presentation}/u src/measurement.ts:23:const maybeEmojiRe = /[\p{Emoji_Presentation}\p{Extended_Pictographic}\p{Regional_Indicator}\uFE0F\u20E3]/u src/measurement.ts:25:const emojiCorrectionCache = new Map() src/measurement.ts:27:export function getMeasureContext(): CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D { src/measurement.ts:43:export function getSegmentMetricCache(font: string): Map { src/measurement.ts:52:export function getSegmentMetrics(seg: string, cache: Map): SegmentMetrics { src/measurement.ts:65:export function getEngineProfile(): EngineProfile { src/measurement.ts:103:export function parseFontSize(font: string): number { src/measurement.ts:108:function getSharedGraphemeSegmenter(): Intl.Segmenter { src/measurement.ts:115:function isEmojiGrapheme(g: string): boolean { src/measurement.ts:119:export function textMayContainEmoji(text: string): boolean { src/measurement.ts:123:function getEmojiCorrection(font: string, fontSize: number): number { src/measurement.ts:153:function countEmojiGraphemes(text: string): number { src/measurement.ts:162:function getEmojiCount(seg: string, metrics: SegmentMetrics): number { src/measurement.ts:169:export function getCorrectedSegmentWidth(seg: string, metrics: SegmentMetrics, emojiCorrection: number): number { src/measurement.ts:174:export function getSegmentGraphemeWidths( src/measurement.ts:193:export function getSegmentGraphemePrefixWidths( src/measurement.ts:214:export function getFontMeasurementState(font: string, needsEmojiCorrection: boolean): { src/measurement.ts:227:export function clearMeasurementCaches(): void { src/analysis.ts:1:export type WhiteSpaceMode = 'normal' | 'pre-wrap' src/analysis.ts:3:export type SegmentBreakKind = src/analysis.ts:13:type SegmentationPiece = { src/analysis.ts:20:export type MergedSegmentation = { src/analysis.ts:28:export type AnalysisChunk = { src/analysis.ts:34:export type TextAnalysis = { normalized: string, chunks: AnalysisChunk[] } & MergedSegmentation src/analysis.ts:36:export type AnalysisProfile = { src/analysis.ts:40:const collapsibleWhitespaceRunRe = /[ \t\n\r\f]+/g src/analysis.ts:41:const needsWhitespaceNormalizationRe = /[\t\n\r\f]| {2,}|^ | $/ src/analysis.ts:43:type WhiteSpaceProfile = { src/analysis.ts:49:function getWhiteSpaceProfile(whiteSpace?: WhiteSpaceMode): WhiteSpaceProfile { src/analysis.ts:56:export function normalizeWhitespaceNormal(text: string): string { src/analysis.ts:69:function normalizeWhitespacePreWrap(text: string): string { src/analysis.ts:79:function getSharedWordSegmenter(): Intl.Segmenter { src/analysis.ts:86:export function clearAnalysisCaches(): void { src/analysis.ts:90:export function setAnalysisLocale(locale?: string): void { src/analysis.ts:97:const arabicScriptRe = /\p{Script=Arabic}/u src/analysis.ts:98:const combiningMarkRe = /\p{M}/u src/analysis.ts:99:const decimalDigitRe = /\p{Nd}/u src/analysis.ts:101:function containsArabicScript(text: string): boolean { src/analysis.ts:105:export function isCJK(s: string): boolean { src/analysis.ts:129:export const kinsokuStart = new Set([ src/analysis.ts:158:export const kinsokuEnd = new Set([ src/analysis.ts:174:const forwardStickyGlue = new Set([ src/analysis.ts:178:export const leftStickyPunctuation = new Set([ src/analysis.ts:197:const arabicNoSpaceTrailingPunctuation = new Set([ src/analysis.ts:204:const myanmarMedialGlue = new Set([ src/analysis.ts:208:const closingQuoteChars = new Set([ src/analysis.ts:219:function isLeftStickyPunctuationSegment(segment: string): boolean { src/analysis.ts:233:function isCJKLineStartProhibitedSegment(segment: string): boolean { src/analysis.ts:240:function isForwardStickyClusterSegment(segment: string): boolean { src/analysis.ts:248:function isEscapedQuoteClusterSegment(segment: string): boolean { src/analysis.ts:261:function splitTrailingForwardStickyCluster(text: string): { head: string, tail: string } | null { src/analysis.ts:285:function isRepeatedSingleCharRun(segment: string, ch: string): boolean { src/analysis.ts:293:function endsWithArabicNoSpacePunctuation(segment: string): boolean { src/analysis.ts:298:function endsWithMyanmarMedialGlue(segment: string): boolean { src/analysis.ts:303:function splitLeadingSpaceAndMarks(segment: string): { space: string, marks: string } | null { src/analysis.ts:312:export function endsWithClosingQuote(text: string): boolean { src/analysis.ts:321:function classifySegmentBreakChar(ch: string, whiteSpaceProfile: WhiteSpaceProfile): SegmentBreakKind { src/analysis.ts:336:function splitSegmentByBreakKind( src/analysis.ts:387:function isTextRunBoundary(kind: SegmentBreakKind): boolean { src/analysis.ts:396:const urlSchemeSegmentRe = /^[A-Za-z][A-Za-z0-9+.-]*:$/ src/analysis.ts:398:function isUrlLikeRunStart(segmentation: MergedSegmentation, index: number): boolean { src/analysis.ts:409:function isUrlQueryBoundarySegment(text: string): boolean { src/analysis.ts:413:function mergeUrlLikeRuns(segmentation: MergedSegmentation): MergedSegmentation { src/analysis.ts:461:function mergeUrlQueryRuns(segmentation: MergedSegmentation): MergedSegmentation { src/analysis.ts:510:const numericJoinerChars = new Set([ src/analysis.ts:516:const asciiPunctuationChainSegmentRe = /^[A-Za-z0-9_]+[,:;]*$/ src/analysis.ts:517:const asciiPunctuationChainTrailingJoinersRe = /[,:;]+$/ src/analysis.ts:519:function segmentContainsDecimalDigit(text: string): boolean { src/analysis.ts:526:function isNumericRunSegment(text: string): boolean { src/analysis.ts:535:function mergeNumericRuns(segmentation: MergedSegmentation): MergedSegmentation { src/analysis.ts:580:function mergeAsciiPunctuationChains(segmentation: MergedSegmentation): MergedSegmentation { src/analysis.ts:629:function splitHyphenatedNumericRuns(segmentation: MergedSegmentation): MergedSegmentation { src/analysis.ts:682:function mergeGlueConnectedTextRuns(segmentation: MergedSegmentation): MergedSegmentation { src/analysis.ts:755:function carryTrailingForwardStickyAcrossCJKBoundary(segmentation: MergedSegmentation): MergedSegmentation { src/analysis.ts:783:function buildMergedSegmentation( src/analysis.ts:946:function compileAnalysisChunks(segmentation: MergedSegmentation, whiteSpaceProfile: WhiteSpaceProfile): AnalysisChunk[] { src/analysis.ts:981:export function analyzeText( src/bidi.ts:7:type BidiType = 'L' | 'R' | 'AL' | 'AN' | 'EN' | 'ES' | 'ET' | 'CS' | src/bidi.ts:10:const baseTypes: BidiType[] = [ src/bidi.ts:32:const arabicTypes: BidiType[] = [ src/bidi.ts:57:function classifyChar(charCode: number): BidiType { src/bidi.ts:65:function computeBidiLevels(str: string): Int8Array | null { src/bidi.ts:164:export function computeSegmentLevels(normalized: string, segStarts: number[]): Int8Array | null { 92:T1cae,Command: /bin/zsh -lc "sed -n '1,260p' src/measurement.ts" Chunk ID: bd68a7 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1795 Output: import { isCJK } from './analysis.js' export type SegmentMetrics = { width: number containsCJK: boolean emojiCount?: number graphemeWidths?: number[] | null graphemePrefixWidths?: number[] | null } export type EngineProfile = { lineFitEpsilon: number carryCJKAfterClosingQuote: boolean preferPrefixWidthsForBreakableRuns: boolean preferEarlySoftHyphenBreak: boolean } let measureContext: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D | null = null const segmentMetricCaches = new Map>() let cachedEngineProfile: EngineProfile | null = null const emojiPresentationRe = /\p{Emoji_Presentation}/u const maybeEmojiRe = /[\p{Emoji_Presentation}\p{Extended_Pictographic}\p{Regional_Indicator}\uFE0F\u20E3]/u let sharedGraphemeSegmenter: Intl.Segmenter | null = null const emojiCorrectionCache = new Map() export function getMeasureContext(): CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D { if (measureContext !== null) return measureContext if (typeof OffscreenCanvas !== 'undefined') { measureContext = new OffscreenCanvas(1, 1).getContext('2d')! return measureContext } if (typeof document !== 'undefined') { measureContext = document.createElement('canvas').getContext('2d')! return measureContext } throw new Error('Text measurement requires OffscreenCanvas or a DOM canvas context.') } export function getSegmentMetricCache(font: string): Map { let cache = segmentMetricCaches.get(font) if (!cache) { cache = new Map() segmentMetricCaches.set(font, cache) } return cache } export function getSegmentMetrics(seg: string, cache: Map): SegmentMetrics { let metrics = cache.get(seg) if (metrics === undefined) { const ctx = getMeasureContext() metrics = { width: ctx.measureText(seg).width, containsCJK: isCJK(seg), } cache.set(seg, metrics) } return metrics } export function getEngineProfile(): EngineProfile { if (cachedEngineProfile !== null) return cachedEngineProfile if (typeof navigator === 'undefined') { cachedEngineProfile = { lineFitEpsilon: 0.005, carryCJKAfterClosingQuote: false, preferPrefixWidthsForBreakableRuns: false, preferEarlySoftHyphenBreak: false, } return cachedEngineProfile } const ua = navigator.userAgent const vendor = navigator.vendor const isSafari = vendor === 'Apple Computer, Inc.' && ua.includes('Safari/') && !ua.includes('Chrome/') && !ua.includes('Chromium/') && !ua.includes('CriOS/') && !ua.includes('FxiOS/') && !ua.includes('EdgiOS/') const isChromium = ua.includes('Chrome/') || ua.includes('Chromium/') || ua.includes('CriOS/') || ua.includes('Edg/') cachedEngineProfile = { lineFitEpsilon: isSafari ? 1 / 64 : 0.005, carryCJKAfterClosingQuote: isChromium, preferPrefixWidthsForBreakableRuns: isSafari, preferEarlySoftHyphenBreak: isSafari, } return cachedEngineProfile } export function parseFontSize(font: string): number { const m = font.match(/(\d+(?:\.\d+)?)\s*px/) return m ? parseFloat(m[1]!) : 16 } function getSharedGraphemeSegmenter(): Intl.Segmenter { if (sharedGraphemeSegmenter === null) { sharedGraphemeSegmenter = new Intl.Segmenter(undefined, { granularity: 'grapheme' }) } return sharedGraphemeSegmenter } function isEmojiGrapheme(g: string): boolean { return emojiPresentationRe.test(g) || g.includes('\uFE0F') } export function textMayContainEmoji(text: string): boolean { return maybeEmojiRe.test(text) } function getEmojiCorrection(font: string, fontSize: number): number { let correction = emojiCorrectionCache.get(font) if (correction !== undefined) return correction const ctx = getMeasureContext() ctx.font = font const canvasW = ctx.measureText('\u{1F600}').width correction = 0 if ( canvasW > fontSize + 0.5 && typeof document !== 'undefined' && document.body !== null ) { const span = document.createElement('span') span.style.font = font span.style.display = 'inline-block' span.style.visibility = 'hidden' span.style.position = 'absolute' span.textContent = '\u{1F600}' document.body.appendChild(span) const domW = span.getBoundingClientRect().width document.body.removeChild(span) if (canvasW - domW > 0.5) { correction = canvasW - domW } } emojiCorrectionCache.set(font, correction) return correction } function countEmojiGraphemes(text: string): number { let count = 0 const graphemeSegmenter = getSharedGraphemeSegmenter() for (const g of graphemeSegmenter.segment(text)) { if (isEmojiGrapheme(g.segment)) count++ } return count } function getEmojiCount(seg: string, metrics: SegmentMetrics): number { if (metrics.emojiCount === undefined) { metrics.emojiCount = countEmojiGraphemes(seg) } return metrics.emojiCount } export function getCorrectedSegmentWidth(seg: string, metrics: SegmentMetrics, emojiCorrection: number): number { if (emojiCorrection === 0) return metrics.width return metrics.width - getEmojiCount(seg, metrics) * emojiCorrection } export function getSegmentGraphemeWidths( seg: string, metrics: SegmentMetrics, cache: Map, emojiCorrection: number, ): number[] | null { if (metrics.graphemeWidths !== undefined) return metrics.graphemeWidths const widths: number[] = [] const graphemeSegmenter = getSharedGraphemeSegmenter() for (const gs of graphemeSegmenter.segment(seg)) { const graphemeMetrics = getSegmentMetrics(gs.segment, cache) widths.push(getCorrectedSegmentWidth(gs.segment, graphemeMetrics, emojiCorrection)) } metrics.graphemeWidths = widths.length > 1 ? widths : null return metrics.graphemeWidths } export function getSegmentGraphemePrefixWidths( seg: string, metrics: SegmentMetrics, cache: Map, emojiCorrection: number, ): number[] | null { if (metrics.graphemePrefixWidths !== undefined) return metrics.graphemePrefixWidths const prefixWidths: number[] = [] const graphemeSegmenter = getSharedGraphemeSegmenter() let prefix = '' for (const gs of graphemeSegmenter.segment(seg)) { prefix += gs.segment const prefixMetrics = getSegmentMetrics(prefix, cache) prefixWidths.push(getCorrectedSegmentWidth(prefix, prefixMetrics, emojiCorrection)) } metrics.graphemePrefixWidths = prefixWidths.length > 1 ? prefixWidths : null return metrics.graphemePrefixWidths } export function getFontMeasurementState(font: string, needsEmojiCorrection: boolean): { cache: Map fontSize: number emojiCorrection: number } { const ctx = getMeasureContext() ctx.font = font const cache = getSegmentMetricCache(font) const fontSize = parseFontSize(font) const emojiCorrection = needsEmojiCorrection ? getEmojiCorrection(font, fontSize) : 0 return { cache, fontSize, emojiCorrection } } export function clearMeasurementCaches(): void { segmentMetricCaches.clear() emojiCorrectionCache.clear() sharedGraphemeSegmenter = null } 93:T1a73,Command: /bin/zsh -lc "sed -n '1,260p' src/bidi.ts" Chunk ID: 0b125e Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1654 Output: // Simplified bidi metadata helper for the rich prepareWithSegments() path, // forked from pdf.js via Sebastian's text-layout. It classifies characters // into bidi types, computes embedding levels, and maps them onto prepared // segments for custom rendering. The line-breaking engine does not consume // these levels. type BidiType = 'L' | 'R' | 'AL' | 'AN' | 'EN' | 'ES' | 'ET' | 'CS' | 'ON' | 'BN' | 'B' | 'S' | 'WS' | 'NSM' const baseTypes: BidiType[] = [ 'BN','BN','BN','BN','BN','BN','BN','BN','BN','S','B','S','WS', 'B','BN','BN','BN','BN','BN','BN','BN','BN','BN','BN','BN','BN', 'BN','BN','B','B','B','S','WS','ON','ON','ET','ET','ET','ON', 'ON','ON','ON','ON','ON','CS','ON','CS','ON','EN','EN','EN', 'EN','EN','EN','EN','EN','EN','EN','ON','ON','ON','ON','ON', 'ON','ON','L','L','L','L','L','L','L','L','L','L','L','L','L', 'L','L','L','L','L','L','L','L','L','L','L','L','L','ON','ON', 'ON','ON','ON','ON','L','L','L','L','L','L','L','L','L','L', 'L','L','L','L','L','L','L','L','L','L','L','L','L','L','L', 'L','ON','ON','ON','ON','BN','BN','BN','BN','BN','BN','B','BN', 'BN','BN','BN','BN','BN','BN','BN','BN','BN','BN','BN','BN', 'BN','BN','BN','BN','BN','BN','BN','BN','BN','BN','BN','BN', 'BN','CS','ON','ET','ET','ET','ET','ON','ON','ON','ON','L','ON', 'ON','ON','ON','ON','ET','ET','EN','EN','ON','L','ON','ON','ON', 'EN','L','ON','ON','ON','ON','ON','L','L','L','L','L','L','L', 'L','L','L','L','L','L','L','L','L','L','L','L','L','L','L', 'L','ON','L','L','L','L','L','L','L','L','L','L','L','L','L', 'L','L','L','L','L','L','L','L','L','L','L','L','L','L','L', 'L','L','L','ON','L','L','L','L','L','L','L','L' ] const arabicTypes: BidiType[] = [ 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'CS','AL','ON','ON','NSM','NSM','NSM','NSM','NSM','NSM','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','AL','AL','AL','NSM','NSM','NSM','NSM','NSM','NSM','NSM', 'NSM','NSM','NSM','NSM','NSM','NSM','NSM','AL','AL','AL','AL', 'AL','AL','AL','AN','AN','AN','AN','AN','AN','AN','AN','AN', 'AN','ET','AN','AN','AL','AL','AL','NSM','AL','AL','AL','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','NSM','NSM','NSM','NSM','NSM','NSM','NSM','NSM','NSM','NSM', 'NSM','NSM','NSM','NSM','NSM','NSM','NSM','NSM','NSM','ON','NSM', 'NSM','NSM','NSM','AL','AL','AL','AL','AL','AL','AL','AL','AL', 'AL','AL','AL','AL','AL','AL','AL','AL','AL' ] function classifyChar(charCode: number): BidiType { if (charCode <= 0x00ff) return baseTypes[charCode]! if (0x0590 <= charCode && charCode <= 0x05f4) return 'R' if (0x0600 <= charCode && charCode <= 0x06ff) return arabicTypes[charCode & 0xff]! if (0x0700 <= charCode && charCode <= 0x08AC) return 'AL' return 'L' } function computeBidiLevels(str: string): Int8Array | null { const len = str.length if (len === 0) return null // eslint-disable-next-line unicorn/no-new-array const types: BidiType[] = new Array(len) let numBidi = 0 for (let i = 0; i < len; i++) { const t = classifyChar(str.charCodeAt(i)) if (t === 'R' || t === 'AL' || t === 'AN') numBidi++ types[i] = t } if (numBidi === 0) return null const startLevel = (len / numBidi) < 0.3 ? 0 : 1 const levels = new Int8Array(len) for (let i = 0; i < len; i++) levels[i] = startLevel const e: BidiType = (startLevel & 1) ? 'R' : 'L' const sor = e // W1-W7 let lastType: BidiType = sor for (let i = 0; i < len; i++) { if (types[i] === 'NSM') types[i] = lastType else lastType = types[i]! } lastType = sor for (let i = 0; i < len; i++) { const t = types[i]! if (t === 'EN') types[i] = lastType === 'AL' ? 'AN' : 'EN' else if (t === 'R' || t === 'L' || t === 'AL') lastType = t } for (let i = 0; i < len; i++) { if (types[i] === 'AL') types[i] = 'R' } for (let i = 1; i < len - 1; i++) { if (types[i] === 'ES' && types[i - 1] === 'EN' && types[i + 1] === 'EN') { types[i] = 'EN' } if ( types[i] === 'CS' && (types[i - 1] === 'EN' || types[i - 1] === 'AN') && types[i + 1] === types[i - 1] ) { types[i] = types[i - 1]! } } for (let i = 0; i < len; i++) { if (types[i] !== 'EN') continue let j for (j = i - 1; j >= 0 && types[j] === 'ET'; j--) types[j] = 'EN' for (j = i + 1; j < len && types[j] === 'ET'; j++) types[j] = 'EN' } for (let i = 0; i < len; i++) { const t = types[i]! if (t === 'WS' || t === 'ES' || t === 'ET' || t === 'CS') types[i] = 'ON' } lastType = sor for (let i = 0; i < len; i++) { const t = types[i]! if (t === 'EN') types[i] = lastType === 'L' ? 'L' : 'EN' else if (t === 'R' || t === 'L') lastType = t } // N1-N2 for (let i = 0; i < len; i++) { if (types[i] !== 'ON') continue let end = i + 1 while (end < len && types[end] === 'ON') end++ const before: BidiType = i > 0 ? types[i - 1]! : sor const after: BidiType = end < len ? types[end]! : sor const bDir: BidiType = before !== 'L' ? 'R' : 'L' const aDir: BidiType = after !== 'L' ? 'R' : 'L' if (bDir === aDir) { for (let j = i; j < end; j++) types[j] = bDir } i = end - 1 } for (let i = 0; i < len; i++) { if (types[i] === 'ON') types[i] = e } // I1-I2 for (let i = 0; i < len; i++) { const t = types[i]! if ((levels[i]! & 1) === 0) { if (t === 'R') levels[i]!++ else if (t === 'AN' || t === 'EN') levels[i]! += 2 } else if (t === 'L' || t === 'AN' || t === 'EN') { levels[i]!++ } } return levels } export function computeSegmentLevels(normalized: string, segStarts: number[]): Int8Array | null { const bidiLevels = computeBidiLevels(normalized) if (bidiLevels === null) return null const segLevels = new Int8Array(segStarts.length) for (let i = 0; i < segStarts.length; i++) { segLevels[i] = bidiLevels[segStarts[i]!]! } return segLevels } 94:T26a4,Command: /bin/zsh -lc "sed -n '1,260p' src/layout.ts" Chunk ID: 2c61dd Wall time: 0.0000 seconds Process exited with code 0 Original token count: 2434 Output: // Text measurement for browser environments using canvas measureText. // // Problem: DOM-based text measurement (getBoundingClientRect, offsetHeight) // forces synchronous layout reflow. When components independently measure text, // each measurement triggers a reflow of the entire document. This creates // read/write interleaving that can cost 30ms+ per frame for 500 text blocks. // // Solution: two-phase measurement centered around canvas measureText. // prepare(text, font) — segments text via Intl.Segmenter, measures each word // via canvas, caches widths, and does one cached DOM calibration read per // font when emoji correction is needed. Call once when text first appears. // layout(prepared, maxWidth, lineHeight) — walks cached word widths with pure // arithmetic to count lines and compute height. Call on every resize. // ~0.0002ms per text. // // i18n: Intl.Segmenter handles CJK (per-character breaking), Thai, Arabic, etc. // Bidi: simplified rich-path metadata for mixed LTR/RTL custom rendering. // Punctuation merging: "better." measured as one unit (matches CSS behavior). // Trailing whitespace: hangs past line edge without triggering breaks (CSS behavior). // overflow-wrap: pre-measured grapheme widths enable character-level word breaking. // // Emoji correction: Chrome/Firefox canvas measures emoji wider than DOM at font // sizes <24px on macOS (Apple Color Emoji). The inflation is constant per emoji // grapheme at a given size, font-independent. Auto-detected by comparing canvas // vs actual DOM emoji width (one cached DOM read per font). Safari canvas and // DOM agree (both wider than fontSize), so correction = 0 there. // // Limitations: // - system-ui font: canvas resolves to different optical variants than DOM on macOS. // Use named fonts (Helvetica, Inter, etc.) for guaranteed accuracy. // See RESEARCH.md "Discovery: system-ui font resolution mismatch". // // Based on Sebastian Markbage's text-layout research (github.com/chenglou/text-layout). import { computeSegmentLevels } from './bidi.js' import { analyzeText, clearAnalysisCaches, endsWithClosingQuote, isCJK, kinsokuEnd, kinsokuStart, leftStickyPunctuation, setAnalysisLocale, type AnalysisChunk, type SegmentBreakKind, type TextAnalysis, type WhiteSpaceMode, } from './analysis.js' import { clearMeasurementCaches, getCorrectedSegmentWidth, getEngineProfile, getFontMeasurementState, getSegmentGraphemePrefixWidths, getSegmentGraphemeWidths, getSegmentMetrics, textMayContainEmoji, } from './measurement.js' import { countPreparedLines, layoutNextLineRange as stepPreparedLineRange, walkPreparedLines, type InternalLayoutLine, } from './line-break.js' let sharedGraphemeSegmenter: Intl.Segmenter | null = null // Rich-path only. Reuses grapheme splits while materializing multiple lines // from the same prepared handle, without pushing that cache into the API. let sharedLineTextCaches = new WeakMap>() function getSharedGraphemeSegmenter(): Intl.Segmenter { if (sharedGraphemeSegmenter === null) { sharedGraphemeSegmenter = new Intl.Segmenter(undefined, { granularity: 'grapheme' }) } return sharedGraphemeSegmenter } // --- Public types --- declare const preparedTextBrand: unique symbol type PreparedCore = { widths: number[] // Segment widths, e.g. [42.5, 4.4, 37.2] lineEndFitAdvances: number[] // Width contribution when a line ends after this segment lineEndPaintAdvances: number[] // Painted width contribution when a line ends after this segment kinds: SegmentBreakKind[] // Break behavior per segment, e.g. ['text', 'space', 'text'] simpleLineWalkFastPath: boolean // Normal text can use the simpler old line walker across all layout APIs segLevels: Int8Array | null // Rich-path bidi metadata for custom rendering; layout() never reads it breakableWidths: (number[] | null)[] // Grapheme widths for overflow-wrap segments, else null breakablePrefixWidths: (number[] | null)[] // Cumulative grapheme prefix widths for narrow browser-policy shims discretionaryHyphenWidth: number // Visible width added when a soft hyphen is chosen as the break tabStopAdvance: number // Absolute advance between tab stops for pre-wrap tab segments chunks: PreparedLineChunk[] // Precompiled hard-break chunks for line walking } // Keep the main prepared handle opaque so the public API does not accidentally // calcify around the current parallel-array representation. export type PreparedText = { readonly [preparedTextBrand]: true } type InternalPreparedText = PreparedText & PreparedCore // Rich/diagnostic variant that still exposes the structural segment data. // Treat this as the unstable escape hatch for experiments and custom rendering. export type PreparedTextWithSegments = InternalPreparedText & { segments: string[] // Segment text aligned with the parallel arrays, e.g. ['hello', ' ', 'world'] } export type LayoutCursor = { segmentIndex: number // Segment index in `segments` graphemeIndex: number // Grapheme index within that segment; `0` at segment boundaries } export type LayoutResult = { lineCount: number // Number of wrapped lines, e.g. 3 height: number // Total block height, e.g. lineCount * lineHeight = 57 } export type LayoutLine = { text: string // Full text content of this line, e.g. 'hello world' width: number // Measured width of this line, e.g. 87.5 start: LayoutCursor // Inclusive start cursor in prepared segments/graphemes end: LayoutCursor // Exclusive end cursor in prepared segments/graphemes } export type LayoutLineRange = { width: number // Measured width of this line, e.g. 87.5 start: LayoutCursor // Inclusive start cursor in prepared segments/graphemes end: LayoutCursor // Exclusive end cursor in prepared segments/graphemes } export type LayoutLinesResult = LayoutResult & { lines: LayoutLine[] // Per-line text/width pairs for custom rendering } export type PrepareProfile = { analysisMs: number measureMs: number totalMs: number analysisSegments: number preparedSegments: number breakableSegments: number } export type PrepareOptions = { whiteSpace?: WhiteSpaceMode } export type PreparedLineChunk = { startSegmentIndex: number endSegmentIndex: number consumedEndSegmentIndex: number } // --- Public API --- function createEmptyPrepared(includeSegments: boolean): InternalPreparedText | PreparedTextWithSegments { if (includeSegments) { return { widths: [], lineEndFitAdvances: [], lineEndPaintAdvances: [], kinds: [], simpleLineWalkFastPath: true, segLevels: null, breakableWidths: [], breakablePrefixWidths: [], discretionaryHyphenWidth: 0, tabStopAdvance: 0, chunks: [], segments: [], } as unknown as PreparedTextWithSegments } return { widths: [], lineEndFitAdvances: [], lineEndPaintAdvances: [], kinds: [], simpleLineWalkFastPath: true, segLevels: null, breakableWidths: [], breakablePrefixWidths: [], discretionaryHyphenWidth: 0, tabStopAdvance: 0, chunks: [], } as unknown as InternalPreparedText } function measureAnalysis( analysis: TextAnalysis, font: string, includeSegments: boolean, ): InternalPreparedText | PreparedTextWithSegments { const graphemeSegmenter = getSharedGraphemeSegmenter() const engineProfile = getEngineProfile() const { cache, emojiCorrection } = getFontMeasurementState( font, textMayContainEmoji(analysis.normalized), ) const discretionaryHyphenWidth = getCorrectedSegmentWidth('-', getSegmentMetrics('-', cache), emojiCorrection) const spaceWidth = getCorrectedSegmentWidth(' ', getSegmentMetrics(' ', cache), emojiCorrection) const tabStopAdvance = spaceWidth * 8 if (analysis.len === 0) return createEmptyPrepared(includeSegments) const widths: number[] = [] const lineEndFitAdvances: number[] = [] const lineEndPaintAdvances: number[] = [] const kinds: SegmentBreakKind[] = [] let simpleLineWalkFastPath = analysis.chunks.length <= 1 const segStarts = includeSegments ? [] as number[] : null const breakableWidths: (number[] | null)[] = [] const breakablePrefixWidths: (number[] | null)[] = [] const segments = includeSegments ? [] as string[] : null const preparedStartByAnalysisIndex = Array.from({ length: analysis.len }) const preparedEndByAnalysisIndex = Array.from({ length: analysis.len }) function pushMeasuredSegment( text: string, width: number, lineEndFitAdvance: number, lineEndPaintAdvance: number, kind: SegmentBreakKind, start: number, breakable: number[] | null, breakablePrefix: number[] | null, ): void { if (kind !== 'text' && kind !== 'space' && kind !== 'zero-width-break') { simpleLineWalkFastPath = false } widths.push(width) lineEndFitAdvances.push(lineEndFitAdvance) lineEndPaintAdvances.push(lineEndPaintAdvance) kinds.push(kind) segStarts?.push(start) breakableWidths.push(breakable) breakablePrefixWidths.push(breakablePrefix) if (segments !== null) segments.push(text) } for (let mi = 0; mi < analysis.len; mi++) { preparedStartByAnalysisIndex[mi] = widths.length const segText = analysis.texts[mi]! const segWordLike = analysis.isWordLike[mi]! const segKind = analysis.kinds[mi]! const segStart = analysis.starts[mi]! if (segKind === 'soft-hyphen') { pushMeasuredSegment( segText, 0, discretionaryHyphenWidth, discretionaryHyphenWidth, segKind, segStart, null, null, ) 95:T2497,Command: /bin/zsh -lc "sed -n '261,520p' src/layout.ts" Chunk ID: e7c526 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 2302 Output: preparedEndByAnalysisIndex[mi] = widths.length continue } if (segKind === 'hard-break') { pushMeasuredSegment(segText, 0, 0, 0, segKind, segStart, null, null) preparedEndByAnalysisIndex[mi] = widths.length continue } if (segKind === 'tab') { pushMeasuredSegment(segText, 0, 0, 0, segKind, segStart, null, null) preparedEndByAnalysisIndex[mi] = widths.length continue } const segMetrics = getSegmentMetrics(segText, cache) if (segKind === 'text' && segMetrics.containsCJK) { let unitText = '' let unitStart = 0 for (const gs of graphemeSegmenter.segment(segText)) { const grapheme = gs.segment if (unitText.length === 0) { unitText = grapheme unitStart = gs.index continue } if ( kinsokuEnd.has(unitText) || kinsokuStart.has(grapheme) || leftStickyPunctuation.has(grapheme) || (engineProfile.carryCJKAfterClosingQuote && isCJK(grapheme) && endsWithClosingQuote(unitText)) ) { unitText += grapheme continue } const unitMetrics = getSegmentMetrics(unitText, cache) const w = getCorrectedSegmentWidth(unitText, unitMetrics, emojiCorrection) pushMeasuredSegment(unitText, w, w, w, 'text', segStart + unitStart, null, null) unitText = grapheme unitStart = gs.index } if (unitText.length > 0) { const unitMetrics = getSegmentMetrics(unitText, cache) const w = getCorrectedSegmentWidth(unitText, unitMetrics, emojiCorrection) pushMeasuredSegment(unitText, w, w, w, 'text', segStart + unitStart, null, null) } preparedEndByAnalysisIndex[mi] = widths.length continue } const w = getCorrectedSegmentWidth(segText, segMetrics, emojiCorrection) const lineEndFitAdvance = segKind === 'space' || segKind === 'preserved-space' || segKind === 'zero-width-break' ? 0 : w const lineEndPaintAdvance = segKind === 'space' || segKind === 'zero-width-break' ? 0 : w if (segWordLike && segText.length > 1) { const graphemeWidths = getSegmentGraphemeWidths(segText, segMetrics, cache, emojiCorrection) const graphemePrefixWidths = engineProfile.preferPrefixWidthsForBreakableRuns ? getSegmentGraphemePrefixWidths(segText, segMetrics, cache, emojiCorrection) : null pushMeasuredSegment( segText, w, lineEndFitAdvance, lineEndPaintAdvance, segKind, segStart, graphemeWidths, graphemePrefixWidths, ) } else { pushMeasuredSegment( segText, w, lineEndFitAdvance, lineEndPaintAdvance, segKind, segStart, null, null, ) } preparedEndByAnalysisIndex[mi] = widths.length } const chunks = mapAnalysisChunksToPreparedChunks(analysis.chunks, preparedStartByAnalysisIndex, preparedEndByAnalysisIndex) const segLevels = segStarts === null ? null : computeSegmentLevels(analysis.normalized, segStarts) if (segments !== null) { return { widths, lineEndFitAdvances, lineEndPaintAdvances, kinds, simpleLineWalkFastPath, segLevels, breakableWidths, breakablePrefixWidths, discretionaryHyphenWidth, tabStopAdvance, chunks, segments, } as unknown as PreparedTextWithSegments } return { widths, lineEndFitAdvances, lineEndPaintAdvances, kinds, simpleLineWalkFastPath, segLevels, breakableWidths, breakablePrefixWidths, discretionaryHyphenWidth, tabStopAdvance, chunks, } as unknown as InternalPreparedText } function mapAnalysisChunksToPreparedChunks( chunks: AnalysisChunk[], preparedStartByAnalysisIndex: number[], preparedEndByAnalysisIndex: number[], ): PreparedLineChunk[] { const preparedChunks: PreparedLineChunk[] = [] for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]! const startSegmentIndex = chunk.startSegmentIndex < preparedStartByAnalysisIndex.length ? preparedStartByAnalysisIndex[chunk.startSegmentIndex]! : preparedEndByAnalysisIndex[preparedEndByAnalysisIndex.length - 1] ?? 0 const endSegmentIndex = chunk.endSegmentIndex < preparedStartByAnalysisIndex.length ? preparedStartByAnalysisIndex[chunk.endSegmentIndex]! : preparedEndByAnalysisIndex[preparedEndByAnalysisIndex.length - 1] ?? 0 const consumedEndSegmentIndex = chunk.consumedEndSegmentIndex < preparedStartByAnalysisIndex.length ? preparedStartByAnalysisIndex[chunk.consumedEndSegmentIndex]! : preparedEndByAnalysisIndex[preparedEndByAnalysisIndex.length - 1] ?? 0 preparedChunks.push({ startSegmentIndex, endSegmentIndex, consumedEndSegmentIndex, }) } return preparedChunks } function prepareInternal( text: string, font: string, includeSegments: boolean, options?: PrepareOptions, ): InternalPreparedText | PreparedTextWithSegments { const analysis = analyzeText(text, getEngineProfile(), options?.whiteSpace) return measureAnalysis(analysis, font, includeSegments) } // Diagnostic-only helper used by the browser benchmark harness to separate the // text-analysis and measurement phases without duplicating the prepare logic. export function profilePrepare(text: string, font: string, options?: PrepareOptions): PrepareProfile { const t0 = performance.now() const analysis = analyzeText(text, getEngineProfile(), options?.whiteSpace) const t1 = performance.now() const prepared = measureAnalysis(analysis, font, false) as InternalPreparedText const t2 = performance.now() let breakableSegments = 0 for (const widths of prepared.breakableWidths) { if (widths !== null) breakableSegments++ } return { analysisMs: t1 - t0, measureMs: t2 - t1, totalMs: t2 - t0, analysisSegments: analysis.len, preparedSegments: prepared.widths.length, breakableSegments, } } // Prepare text for layout. Segments the text, measures each segment via canvas, // and stores the widths for fast relayout at any width. Call once per text block // (e.g. when a comment first appears). The result is width-independent — the // same PreparedText can be laid out at any maxWidth and lineHeight via layout(). // // Steps: // 1. Normalize collapsible whitespace (CSS white-space: normal behavior) // 2. Segment via Intl.Segmenter (handles CJK, Thai, etc.) // 3. Merge punctuation into preceding word ("better." as one unit) // 4. Split CJK words into individual graphemes (per-character line breaks) // 5. Measure each segment via canvas measureText, cache by (segment, font) // 6. Pre-measure graphemes of long words (for overflow-wrap: break-word) // 7. Correct emoji canvas inflation (auto-detected per font size) // 8. Optionally compute rich-path bidi metadata for custom renderers export function prepare(text: string, font: string, options?: PrepareOptions): PreparedText { return prepareInternal(text, font, false, options) as PreparedText } // Rich variant used by callers that need enough information to render the // laid-out lines themselves. export function prepareWithSegments(text: string, font: string, options?: PrepareOptions): PreparedTextWithSegments { return prepareInternal(text, font, true, options) as PreparedTextWithSegments } function getInternalPrepared(prepared: PreparedText): InternalPreparedText { return prepared as InternalPreparedText } // Layout prepared text at a given max width and caller-provided lineHeight. // Pure arithmetic on cached widths — no canvas calls, no DOM reads, no string // operations, no allocations. // ~0.0002ms per text block. Call on every resize. // // Line breaking rules (matching CSS white-space: normal + overflow-wrap: break-word): // - Break before any non-space segment that would overflow the line // - Trailing whitespace hangs past the line edge (doesn't trigger breaks) // - Segments wider than maxWidth are broken at grapheme boundaries export function layout(prepared: PreparedText, maxWidth: number, lineHeight: number): LayoutResult { // Keep the resize hot path specialized. `layoutWithLines()` shares the same // break semantics but also tracks line ranges; the extra bookkeeping is too // expensive to pay on every hot-path `layout()` call. const lineCount = countPreparedLines(getInternalPrepared(prepared), maxWidth) return { lineCount, height: lineCount * lineHeight } } function getSegmentGraphemes( segmentIndex: number, segments: string[], cache: Map, ): string[] { let graphemes = cache.get(segmentIndex) if (graphemes !== undefined) return graphemes graphemes = [] const graphemeSegmenter = getSharedGraphemeSegmenter() for (const gs of graphemeSegmenter.segment(segments[segmentIndex]!)) { graphemes.push(gs.segment) } cache.set(segmentIndex, graphemes) return graphemes } function getLineTextCache(prepared: PreparedTextWithSegments): Map { 96:T157d,Command: /bin/zsh -lc "sed -n '521,780p' src/layout.ts" Chunk ID: a944f3 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1335 Output: let cache = sharedLineTextCaches.get(prepared) if (cache !== undefined) return cache cache = new Map() sharedLineTextCaches.set(prepared, cache) return cache } function lineHasDiscretionaryHyphen( kinds: SegmentBreakKind[], startSegmentIndex: number, startGraphemeIndex: number, endSegmentIndex: number, ): boolean { return ( endSegmentIndex > 0 && kinds[endSegmentIndex - 1] === 'soft-hyphen' && !(startSegmentIndex === endSegmentIndex && startGraphemeIndex > 0) ) } function buildLineTextFromRange( segments: string[], kinds: SegmentBreakKind[], cache: Map, startSegmentIndex: number, startGraphemeIndex: number, endSegmentIndex: number, endGraphemeIndex: number, ): string { let text = '' const endsWithDiscretionaryHyphen = lineHasDiscretionaryHyphen( kinds, startSegmentIndex, startGraphemeIndex, endSegmentIndex, ) for (let i = startSegmentIndex; i < endSegmentIndex; i++) { if (kinds[i] === 'soft-hyphen' || kinds[i] === 'hard-break') continue if (i === startSegmentIndex && startGraphemeIndex > 0) { text += getSegmentGraphemes(i, segments, cache).slice(startGraphemeIndex).join('') } else { text += segments[i]! } } if (endGraphemeIndex > 0) { if (endsWithDiscretionaryHyphen) text += '-' text += getSegmentGraphemes(endSegmentIndex, segments, cache).slice( startSegmentIndex === endSegmentIndex ? startGraphemeIndex : 0, endGraphemeIndex, ).join('') } else if (endsWithDiscretionaryHyphen) { text += '-' } return text } function createLayoutLine( prepared: PreparedTextWithSegments, cache: Map, width: number, startSegmentIndex: number, startGraphemeIndex: number, endSegmentIndex: number, endGraphemeIndex: number, ): LayoutLine { return { text: buildLineTextFromRange( prepared.segments, prepared.kinds, cache, startSegmentIndex, startGraphemeIndex, endSegmentIndex, endGraphemeIndex, ), width, start: { segmentIndex: startSegmentIndex, graphemeIndex: startGraphemeIndex, }, end: { segmentIndex: endSegmentIndex, graphemeIndex: endGraphemeIndex, }, } } function materializeLayoutLine( prepared: PreparedTextWithSegments, cache: Map, line: InternalLayoutLine, ): LayoutLine { return createLayoutLine( prepared, cache, line.width, line.startSegmentIndex, line.startGraphemeIndex, line.endSegmentIndex, line.endGraphemeIndex, ) } function toLayoutLineRange(line: InternalLayoutLine): LayoutLineRange { return { width: line.width, start: { segmentIndex: line.startSegmentIndex, graphemeIndex: line.startGraphemeIndex, }, end: { segmentIndex: line.endSegmentIndex, graphemeIndex: line.endGraphemeIndex, }, } } function stepLineRange( prepared: PreparedTextWithSegments, start: LayoutCursor, maxWidth: number, ): LayoutLineRange | null { const line = stepPreparedLineRange(prepared, start, maxWidth) if (line === null) return null return toLayoutLineRange(line) } function materializeLine( prepared: PreparedTextWithSegments, line: LayoutLineRange, ): LayoutLine { return createLayoutLine( prepared, getLineTextCache(prepared), line.width, line.start.segmentIndex, line.start.graphemeIndex, line.end.segmentIndex, line.end.graphemeIndex, ) } // Batch low-level line geometry pass. This is the non-materializing counterpart // to layoutWithLines(), useful for shrinkwrap and other aggregate geometry work. export function walkLineRanges( prepared: PreparedTextWithSegments, maxWidth: number, onLine: (line: LayoutLineRange) => void, ): number { if (prepared.widths.length === 0) return 0 return walkPreparedLines(getInternalPrepared(prepared), maxWidth, line => { onLine(toLayoutLineRange(line)) }) } export function layoutNextLine( prepared: PreparedTextWithSegments, start: LayoutCursor, maxWidth: number, ): LayoutLine | null { const line = stepLineRange(prepared, start, maxWidth) if (line === null) return null return materializeLine(prepared, line) } // Rich layout API for callers that want the actual line contents and widths. // Caller still supplies lineHeight at layout time. Mirrors layout()'s break // decisions, but keeps extra per-line bookkeeping so it should stay off the // resize hot path. export function layoutWithLines(prepared: PreparedTextWithSegments, maxWidth: number, lineHeight: number): LayoutLinesResult { const lines: LayoutLine[] = [] if (prepared.widths.length === 0) return { lineCount: 0, height: 0, lines } const graphemeCache = getLineTextCache(prepared) const lineCount = walkPreparedLines(getInternalPrepared(prepared), maxWidth, line => { lines.push(materializeLayoutLine(prepared, graphemeCache, line)) }) return { lineCount, height: lineCount * lineHeight, lines } } export function clearCache(): void { clearAnalysisCaches() sharedGraphemeSegmenter = null sharedLineTextCaches = new WeakMap>() clearMeasurementCaches() } export function setLocale(locale?: string): void { setAnalysisLocale(locale) clearCache() } 97:T1d67,Command: /bin/zsh -lc "sed -n '1,260p' src/line-break.ts" Chunk ID: 99c16f Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1841 Output: import type { SegmentBreakKind } from './analysis.js' import { getEngineProfile } from './measurement.js' export type LineBreakCursor = { segmentIndex: number graphemeIndex: number } export type PreparedLineBreakData = { widths: number[] lineEndFitAdvances: number[] lineEndPaintAdvances: number[] kinds: SegmentBreakKind[] simpleLineWalkFastPath: boolean breakableWidths: (number[] | null)[] breakablePrefixWidths: (number[] | null)[] discretionaryHyphenWidth: number tabStopAdvance: number chunks: { startSegmentIndex: number endSegmentIndex: number consumedEndSegmentIndex: number }[] } export type InternalLayoutLine = { startSegmentIndex: number startGraphemeIndex: number endSegmentIndex: number endGraphemeIndex: number width: number } function canBreakAfter(kind: SegmentBreakKind): boolean { return ( kind === 'space' || kind === 'preserved-space' || kind === 'tab' || kind === 'zero-width-break' || kind === 'soft-hyphen' ) } function isSimpleCollapsibleSpace(kind: SegmentBreakKind): boolean { return kind === 'space' } function getTabAdvance(lineWidth: number, tabStopAdvance: number): number { if (tabStopAdvance <= 0) return 0 const remainder = lineWidth % tabStopAdvance if (Math.abs(remainder) <= 1e-6) return tabStopAdvance return tabStopAdvance - remainder } function getBreakableAdvance( graphemeWidths: number[], graphemePrefixWidths: number[] | null, graphemeIndex: number, preferPrefixWidths: boolean, ): number { if (!preferPrefixWidths || graphemePrefixWidths === null) { return graphemeWidths[graphemeIndex]! } return graphemePrefixWidths[graphemeIndex]! - (graphemeIndex > 0 ? graphemePrefixWidths[graphemeIndex - 1]! : 0) } function fitSoftHyphenBreak( graphemeWidths: number[], initialWidth: number, maxWidth: number, lineFitEpsilon: number, discretionaryHyphenWidth: number, cumulativeWidths: boolean, ): { fitCount: number, fittedWidth: number } { let fitCount = 0 let fittedWidth = initialWidth while (fitCount < graphemeWidths.length) { const nextWidth = cumulativeWidths ? initialWidth + graphemeWidths[fitCount]! : fittedWidth + graphemeWidths[fitCount]! const nextLineWidth = fitCount + 1 < graphemeWidths.length ? nextWidth + discretionaryHyphenWidth : nextWidth if (nextLineWidth > maxWidth + lineFitEpsilon) break fittedWidth = nextWidth fitCount++ } return { fitCount, fittedWidth } } function findChunkIndexForStart(prepared: PreparedLineBreakData, segmentIndex: number): number { for (let i = 0; i < prepared.chunks.length; i++) { const chunk = prepared.chunks[i]! if (segmentIndex < chunk.consumedEndSegmentIndex) return i } return -1 } export function normalizeLineStart( prepared: PreparedLineBreakData, start: LineBreakCursor, ): LineBreakCursor | null { let segmentIndex = start.segmentIndex const graphemeIndex = start.graphemeIndex if (segmentIndex >= prepared.widths.length) return null if (graphemeIndex > 0) return start const chunkIndex = findChunkIndexForStart(prepared, segmentIndex) if (chunkIndex < 0) return null const chunk = prepared.chunks[chunkIndex]! if (chunk.startSegmentIndex === chunk.endSegmentIndex && segmentIndex === chunk.startSegmentIndex) { return { segmentIndex, graphemeIndex: 0 } } if (segmentIndex < chunk.startSegmentIndex) segmentIndex = chunk.startSegmentIndex while (segmentIndex < chunk.endSegmentIndex) { const kind = prepared.kinds[segmentIndex]! if (kind !== 'space' && kind !== 'zero-width-break' && kind !== 'soft-hyphen') { return { segmentIndex, graphemeIndex: 0 } } segmentIndex++ } if (chunk.consumedEndSegmentIndex >= prepared.widths.length) return null return { segmentIndex: chunk.consumedEndSegmentIndex, graphemeIndex: 0 } } export function countPreparedLines(prepared: PreparedLineBreakData, maxWidth: number): number { if (prepared.simpleLineWalkFastPath) { return countPreparedLinesSimple(prepared, maxWidth) } return walkPreparedLines(prepared, maxWidth) } function countPreparedLinesSimple(prepared: PreparedLineBreakData, maxWidth: number): number { const { widths, kinds, breakableWidths, breakablePrefixWidths } = prepared if (widths.length === 0) return 0 const engineProfile = getEngineProfile() const lineFitEpsilon = engineProfile.lineFitEpsilon let lineCount = 0 let lineW = 0 let hasContent = false function placeOnFreshLine(segmentIndex: number): void { const w = widths[segmentIndex]! if (w > maxWidth && breakableWidths[segmentIndex] !== null) { const gWidths = breakableWidths[segmentIndex]! const gPrefixWidths = breakablePrefixWidths[segmentIndex] ?? null lineW = 0 for (let g = 0; g < gWidths.length; g++) { const gw = getBreakableAdvance( gWidths, gPrefixWidths, g, engineProfile.preferPrefixWidthsForBreakableRuns, ) if (lineW > 0 && lineW + gw > maxWidth + lineFitEpsilon) { lineCount++ lineW = gw } else { if (lineW === 0) lineCount++ lineW += gw } } } else { lineW = w lineCount++ } hasContent = true } for (let i = 0; i < widths.length; i++) { const w = widths[i]! const kind = kinds[i]! if (!hasContent) { placeOnFreshLine(i) continue } const newW = lineW + w if (newW > maxWidth + lineFitEpsilon) { if (isSimpleCollapsibleSpace(kind)) continue lineW = 0 hasContent = false placeOnFreshLine(i) continue } lineW = newW } if (!hasContent) return lineCount + 1 return lineCount } function walkPreparedLinesSimple( prepared: PreparedLineBreakData, maxWidth: number, onLine?: (line: InternalLayoutLine) => void, ): number { const { widths, kinds, breakableWidths, breakablePrefixWidths } = prepared if (widths.length === 0) return 0 const engineProfile = getEngineProfile() const lineFitEpsilon = engineProfile.lineFitEpsilon let lineCount = 0 let lineW = 0 let hasContent = false let lineStartSegmentIndex = 0 let lineStartGraphemeIndex = 0 let lineEndSegmentIndex = 0 let lineEndGraphemeIndex = 0 let pendingBreakSegmentIndex = -1 let pendingBreakPaintWidth = 0 function clearPendingBreak(): void { pendingBreakSegmentIndex = -1 pendingBreakPaintWidth = 0 } function emitCurrentLine( endSegmentIndex = lineEndSegmentIndex, endGraphemeIndex = lineEndGraphemeIndex, width = lineW, ): void { lineCount++ onLine?.({ startSegmentIndex: lineStartSegmentIndex, startGraphemeIndex: lineStartGraphemeIndex, endSegmentIndex, endGraphemeIndex, width, }) lineW = 0 hasContent = false clearPendingBreak() } function startLineAtSegment(segmentIndex: number, width: number): void { hasContent = true lineStartSegmentIndex = segmentIndex lineStartGraphemeIndex = 0 lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 lineW = width } function startLineAtGrapheme(segmentIndex: number, graphemeIndex: number, width: number): void { hasContent = true lineStartSegmentIndex = segmentIndex lineStartGraphemeIndex = graphemeIndex 98:T1dfb,Command: /bin/zsh -lc "sed -n '261,520p' src/line-break.ts" Chunk ID: 7bbea1 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1878 Output: lineEndSegmentIndex = segmentIndex lineEndGraphemeIndex = graphemeIndex + 1 lineW = width } function appendWholeSegment(segmentIndex: number, width: number): void { if (!hasContent) { startLineAtSegment(segmentIndex, width) return } lineW += width lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 } function updatePendingBreak(segmentIndex: number, segmentWidth: number): void { if (!canBreakAfter(kinds[segmentIndex]!)) return pendingBreakSegmentIndex = segmentIndex + 1 pendingBreakPaintWidth = lineW - segmentWidth } function appendBreakableSegment(segmentIndex: number): void { appendBreakableSegmentFrom(segmentIndex, 0) } function appendBreakableSegmentFrom(segmentIndex: number, startGraphemeIndex: number): void { const gWidths = breakableWidths[segmentIndex]! const gPrefixWidths = breakablePrefixWidths[segmentIndex] ?? null for (let g = startGraphemeIndex; g < gWidths.length; g++) { const gw = getBreakableAdvance( gWidths, gPrefixWidths, g, engineProfile.preferPrefixWidthsForBreakableRuns, ) if (!hasContent) { startLineAtGrapheme(segmentIndex, g, gw) continue } if (lineW + gw > maxWidth + lineFitEpsilon) { emitCurrentLine() startLineAtGrapheme(segmentIndex, g, gw) } else { lineW += gw lineEndSegmentIndex = segmentIndex lineEndGraphemeIndex = g + 1 } } if (hasContent && lineEndSegmentIndex === segmentIndex && lineEndGraphemeIndex === gWidths.length) { lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 } } let i = 0 while (i < widths.length) { const w = widths[i]! const kind = kinds[i]! if (!hasContent) { if (w > maxWidth && breakableWidths[i] !== null) { appendBreakableSegment(i) } else { startLineAtSegment(i, w) } updatePendingBreak(i, w) i++ continue } const newW = lineW + w if (newW > maxWidth + lineFitEpsilon) { if (canBreakAfter(kind)) { appendWholeSegment(i, w) emitCurrentLine(i + 1, 0, lineW - w) i++ continue } if (pendingBreakSegmentIndex >= 0) { if ( lineEndSegmentIndex > pendingBreakSegmentIndex || (lineEndSegmentIndex === pendingBreakSegmentIndex && lineEndGraphemeIndex > 0) ) { emitCurrentLine() continue } emitCurrentLine(pendingBreakSegmentIndex, 0, pendingBreakPaintWidth) continue } if (w > maxWidth && breakableWidths[i] !== null) { emitCurrentLine() appendBreakableSegment(i) i++ continue } emitCurrentLine() continue } appendWholeSegment(i, w) updatePendingBreak(i, w) i++ } if (hasContent) emitCurrentLine() return lineCount } export function walkPreparedLines( prepared: PreparedLineBreakData, maxWidth: number, onLine?: (line: InternalLayoutLine) => void, ): number { if (prepared.simpleLineWalkFastPath) { return walkPreparedLinesSimple(prepared, maxWidth, onLine) } const { widths, lineEndFitAdvances, lineEndPaintAdvances, kinds, breakableWidths, breakablePrefixWidths, discretionaryHyphenWidth, tabStopAdvance, chunks, } = prepared if (widths.length === 0 || chunks.length === 0) return 0 const engineProfile = getEngineProfile() const lineFitEpsilon = engineProfile.lineFitEpsilon let lineCount = 0 let lineW = 0 let hasContent = false let lineStartSegmentIndex = 0 let lineStartGraphemeIndex = 0 let lineEndSegmentIndex = 0 let lineEndGraphemeIndex = 0 let pendingBreakSegmentIndex = -1 let pendingBreakFitWidth = 0 let pendingBreakPaintWidth = 0 let pendingBreakKind: SegmentBreakKind | null = null function clearPendingBreak(): void { pendingBreakSegmentIndex = -1 pendingBreakFitWidth = 0 pendingBreakPaintWidth = 0 pendingBreakKind = null } function emitCurrentLine( endSegmentIndex = lineEndSegmentIndex, endGraphemeIndex = lineEndGraphemeIndex, width = lineW, ): void { lineCount++ onLine?.({ startSegmentIndex: lineStartSegmentIndex, startGraphemeIndex: lineStartGraphemeIndex, endSegmentIndex, endGraphemeIndex, width, }) lineW = 0 hasContent = false clearPendingBreak() } function startLineAtSegment(segmentIndex: number, width: number): void { hasContent = true lineStartSegmentIndex = segmentIndex lineStartGraphemeIndex = 0 lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 lineW = width } function startLineAtGrapheme(segmentIndex: number, graphemeIndex: number, width: number): void { hasContent = true lineStartSegmentIndex = segmentIndex lineStartGraphemeIndex = graphemeIndex lineEndSegmentIndex = segmentIndex lineEndGraphemeIndex = graphemeIndex + 1 lineW = width } function appendWholeSegment(segmentIndex: number, width: number): void { if (!hasContent) { startLineAtSegment(segmentIndex, width) return } lineW += width lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 } function updatePendingBreakForWholeSegment(segmentIndex: number, segmentWidth: number): void { if (!canBreakAfter(kinds[segmentIndex]!)) return const fitAdvance = kinds[segmentIndex] === 'tab' ? 0 : lineEndFitAdvances[segmentIndex]! const paintAdvance = kinds[segmentIndex] === 'tab' ? segmentWidth : lineEndPaintAdvances[segmentIndex]! pendingBreakSegmentIndex = segmentIndex + 1 pendingBreakFitWidth = lineW - segmentWidth + fitAdvance pendingBreakPaintWidth = lineW - segmentWidth + paintAdvance pendingBreakKind = kinds[segmentIndex]! } function appendBreakableSegment(segmentIndex: number): void { appendBreakableSegmentFrom(segmentIndex, 0) } function appendBreakableSegmentFrom(segmentIndex: number, startGraphemeIndex: number): void { const gWidths = breakableWidths[segmentIndex]! const gPrefixWidths = breakablePrefixWidths[segmentIndex] ?? null for (let g = startGraphemeIndex; g < gWidths.length; g++) { const gw = getBreakableAdvance( gWidths, gPrefixWidths, g, engineProfile.preferPrefixWidthsForBreakableRuns, ) if (!hasContent) { startLineAtGrapheme(segmentIndex, g, gw) continue } if (lineW + gw > maxWidth + lineFitEpsilon) { emitCurrentLine() startLineAtGrapheme(segmentIndex, g, gw) } else { lineW += gw lineEndSegmentIndex = segmentIndex lineEndGraphemeIndex = g + 1 } } if (hasContent && lineEndSegmentIndex === segmentIndex && lineEndGraphemeIndex === gWidths.length) { lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 } } function continueSoftHyphenBreakableSegment(segmentIndex: number): boolean { if (pendingBreakKind !== 'soft-hyphen') return false const gWidths = breakableWidths[segmentIndex]! if (gWidths === null) return false const fitWidths = engineProfile.preferPrefixWidthsForBreakableRuns ? breakablePrefixWidths[segmentIndex] ?? gWidths : gWidths const usesPrefixWidths = fitWidths !== gWidths const { fitCount, fittedWidth } = fitSoftHyphenBreak( fitWidths, 99:T1ecd,Command: /bin/zsh -lc "sed -n '521,780p' src/line-break.ts" Chunk ID: 2a4a71 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1930 Output: lineW, maxWidth, lineFitEpsilon, discretionaryHyphenWidth, usesPrefixWidths, ) if (fitCount === 0) return false lineW = fittedWidth lineEndSegmentIndex = segmentIndex lineEndGraphemeIndex = fitCount clearPendingBreak() if (fitCount === gWidths.length) { lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 return true } emitCurrentLine( segmentIndex, fitCount, fittedWidth + discretionaryHyphenWidth, ) appendBreakableSegmentFrom(segmentIndex, fitCount) return true } function emitEmptyChunk(chunk: { startSegmentIndex: number, consumedEndSegmentIndex: number }): void { lineCount++ onLine?.({ startSegmentIndex: chunk.startSegmentIndex, startGraphemeIndex: 0, endSegmentIndex: chunk.consumedEndSegmentIndex, endGraphemeIndex: 0, width: 0, }) clearPendingBreak() } for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) { const chunk = chunks[chunkIndex]! if (chunk.startSegmentIndex === chunk.endSegmentIndex) { emitEmptyChunk(chunk) continue } hasContent = false lineW = 0 lineStartSegmentIndex = chunk.startSegmentIndex lineStartGraphemeIndex = 0 lineEndSegmentIndex = chunk.startSegmentIndex lineEndGraphemeIndex = 0 clearPendingBreak() let i = chunk.startSegmentIndex while (i < chunk.endSegmentIndex) { const kind = kinds[i]! const w = kind === 'tab' ? getTabAdvance(lineW, tabStopAdvance) : widths[i]! if (kind === 'soft-hyphen') { if (hasContent) { lineEndSegmentIndex = i + 1 lineEndGraphemeIndex = 0 pendingBreakSegmentIndex = i + 1 pendingBreakFitWidth = lineW + discretionaryHyphenWidth pendingBreakPaintWidth = lineW + discretionaryHyphenWidth pendingBreakKind = kind } i++ continue } if (!hasContent) { if (w > maxWidth && breakableWidths[i] !== null) { appendBreakableSegment(i) } else { startLineAtSegment(i, w) } updatePendingBreakForWholeSegment(i, w) i++ continue } const newW = lineW + w if (newW > maxWidth + lineFitEpsilon) { const currentBreakFitWidth = lineW + (kind === 'tab' ? 0 : lineEndFitAdvances[i]!) const currentBreakPaintWidth = lineW + (kind === 'tab' ? w : lineEndPaintAdvances[i]!) if ( pendingBreakKind === 'soft-hyphen' && engineProfile.preferEarlySoftHyphenBreak && pendingBreakFitWidth <= maxWidth + lineFitEpsilon ) { emitCurrentLine(pendingBreakSegmentIndex, 0, pendingBreakPaintWidth) continue } if (pendingBreakKind === 'soft-hyphen' && continueSoftHyphenBreakableSegment(i)) { i++ continue } if (canBreakAfter(kind) && currentBreakFitWidth <= maxWidth + lineFitEpsilon) { appendWholeSegment(i, w) emitCurrentLine(i + 1, 0, currentBreakPaintWidth) i++ continue } if (pendingBreakSegmentIndex >= 0 && pendingBreakFitWidth <= maxWidth + lineFitEpsilon) { if ( lineEndSegmentIndex > pendingBreakSegmentIndex || (lineEndSegmentIndex === pendingBreakSegmentIndex && lineEndGraphemeIndex > 0) ) { emitCurrentLine() continue } const nextSegmentIndex = pendingBreakSegmentIndex emitCurrentLine(nextSegmentIndex, 0, pendingBreakPaintWidth) i = nextSegmentIndex continue } if (w > maxWidth && breakableWidths[i] !== null) { emitCurrentLine() appendBreakableSegment(i) i++ continue } emitCurrentLine() continue } appendWholeSegment(i, w) updatePendingBreakForWholeSegment(i, w) i++ } if (hasContent) { const finalPaintWidth = pendingBreakSegmentIndex === chunk.consumedEndSegmentIndex ? pendingBreakPaintWidth : lineW emitCurrentLine(chunk.consumedEndSegmentIndex, 0, finalPaintWidth) } } return lineCount } export function layoutNextLineRange( prepared: PreparedLineBreakData, start: LineBreakCursor, maxWidth: number, ): InternalLayoutLine | null { const normalizedStart = normalizeLineStart(prepared, start) if (normalizedStart === null) return null if (prepared.simpleLineWalkFastPath) { return layoutNextLineRangeSimple(prepared, normalizedStart, maxWidth) } const chunkIndex = findChunkIndexForStart(prepared, normalizedStart.segmentIndex) if (chunkIndex < 0) return null const chunk = prepared.chunks[chunkIndex]! if (chunk.startSegmentIndex === chunk.endSegmentIndex) { return { startSegmentIndex: chunk.startSegmentIndex, startGraphemeIndex: 0, endSegmentIndex: chunk.consumedEndSegmentIndex, endGraphemeIndex: 0, width: 0, } } const { widths, lineEndFitAdvances, lineEndPaintAdvances, kinds, breakableWidths, breakablePrefixWidths, discretionaryHyphenWidth, tabStopAdvance, } = prepared const engineProfile = getEngineProfile() const lineFitEpsilon = engineProfile.lineFitEpsilon let lineW = 0 let hasContent = false const lineStartSegmentIndex = normalizedStart.segmentIndex const lineStartGraphemeIndex = normalizedStart.graphemeIndex let lineEndSegmentIndex = lineStartSegmentIndex let lineEndGraphemeIndex = lineStartGraphemeIndex let pendingBreakSegmentIndex = -1 let pendingBreakFitWidth = 0 let pendingBreakPaintWidth = 0 let pendingBreakKind: SegmentBreakKind | null = null function clearPendingBreak(): void { pendingBreakSegmentIndex = -1 pendingBreakFitWidth = 0 pendingBreakPaintWidth = 0 pendingBreakKind = null } function finishLine( endSegmentIndex = lineEndSegmentIndex, endGraphemeIndex = lineEndGraphemeIndex, width = lineW, ): InternalLayoutLine | null { if (!hasContent) return null return { startSegmentIndex: lineStartSegmentIndex, startGraphemeIndex: lineStartGraphemeIndex, endSegmentIndex, endGraphemeIndex, width, } } function startLineAtSegment(segmentIndex: number, width: number): void { hasContent = true lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 lineW = width } function startLineAtGrapheme(segmentIndex: number, graphemeIndex: number, width: number): void { hasContent = true lineEndSegmentIndex = segmentIndex lineEndGraphemeIndex = graphemeIndex + 1 lineW = width } function appendWholeSegment(segmentIndex: number, width: number): void { if (!hasContent) { startLineAtSegment(segmentIndex, width) return } lineW += width lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 } function updatePendingBreakForWholeSegment(segmentIndex: number, segmentWidth: number): void { if (!canBreakAfter(kinds[segmentIndex]!)) return const fitAdvance = kinds[segmentIndex] === 'tab' ? 0 : lineEndFitAdvances[segmentIndex]! const paintAdvance = kinds[segmentIndex] === 'tab' ? segmentWidth : lineEndPaintAdvances[segmentIndex]! pendingBreakSegmentIndex = segmentIndex + 1 pendingBreakFitWidth = lineW - segmentWidth + fitAdvance pendingBreakPaintWidth = lineW - segmentWidth + paintAdvance pendingBreakKind = kinds[segmentIndex]! } function appendBreakableSegmentFrom(segmentIndex: number, startGraphemeIndex: number): InternalLayoutLine | null { 9a:T25c1,Command: /bin/zsh -lc "sed -n '781,1084p' src/line-break.ts" Chunk ID: 76e986 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 2375 Output: const gWidths = breakableWidths[segmentIndex]! const gPrefixWidths = breakablePrefixWidths[segmentIndex] ?? null for (let g = startGraphemeIndex; g < gWidths.length; g++) { const gw = getBreakableAdvance( gWidths, gPrefixWidths, g, engineProfile.preferPrefixWidthsForBreakableRuns, ) if (!hasContent) { startLineAtGrapheme(segmentIndex, g, gw) continue } if (lineW + gw > maxWidth + lineFitEpsilon) { return finishLine() } lineW += gw lineEndSegmentIndex = segmentIndex lineEndGraphemeIndex = g + 1 } if (hasContent && lineEndSegmentIndex === segmentIndex && lineEndGraphemeIndex === gWidths.length) { lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 } return null } function maybeFinishAtSoftHyphen(segmentIndex: number): InternalLayoutLine | null { if (pendingBreakKind !== 'soft-hyphen' || pendingBreakSegmentIndex < 0) return null const gWidths = breakableWidths[segmentIndex] ?? null if (gWidths !== null) { const fitWidths = engineProfile.preferPrefixWidthsForBreakableRuns ? breakablePrefixWidths[segmentIndex] ?? gWidths : gWidths const usesPrefixWidths = fitWidths !== gWidths const { fitCount, fittedWidth } = fitSoftHyphenBreak( fitWidths, lineW, maxWidth, lineFitEpsilon, discretionaryHyphenWidth, usesPrefixWidths, ) if (fitCount === gWidths.length) { lineW = fittedWidth lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 clearPendingBreak() return null } if (fitCount > 0) { return finishLine( segmentIndex, fitCount, fittedWidth + discretionaryHyphenWidth, ) } } if (pendingBreakFitWidth <= maxWidth + lineFitEpsilon) { return finishLine(pendingBreakSegmentIndex, 0, pendingBreakPaintWidth) } return null } for (let i = normalizedStart.segmentIndex; i < chunk.endSegmentIndex; i++) { const kind = kinds[i]! const startGraphemeIndex = i === normalizedStart.segmentIndex ? normalizedStart.graphemeIndex : 0 const w = kind === 'tab' ? getTabAdvance(lineW, tabStopAdvance) : widths[i]! if (kind === 'soft-hyphen' && startGraphemeIndex === 0) { if (hasContent) { lineEndSegmentIndex = i + 1 lineEndGraphemeIndex = 0 pendingBreakSegmentIndex = i + 1 pendingBreakFitWidth = lineW + discretionaryHyphenWidth pendingBreakPaintWidth = lineW + discretionaryHyphenWidth pendingBreakKind = kind } continue } if (!hasContent) { if (startGraphemeIndex > 0) { const line = appendBreakableSegmentFrom(i, startGraphemeIndex) if (line !== null) return line } else if (w > maxWidth && breakableWidths[i] !== null) { const line = appendBreakableSegmentFrom(i, 0) if (line !== null) return line } else { startLineAtSegment(i, w) } updatePendingBreakForWholeSegment(i, w) continue } const newW = lineW + w if (newW > maxWidth + lineFitEpsilon) { const currentBreakFitWidth = lineW + (kind === 'tab' ? 0 : lineEndFitAdvances[i]!) const currentBreakPaintWidth = lineW + (kind === 'tab' ? w : lineEndPaintAdvances[i]!) if ( pendingBreakKind === 'soft-hyphen' && engineProfile.preferEarlySoftHyphenBreak && pendingBreakFitWidth <= maxWidth + lineFitEpsilon ) { return finishLine(pendingBreakSegmentIndex, 0, pendingBreakPaintWidth) } const softBreakLine = maybeFinishAtSoftHyphen(i) if (softBreakLine !== null) return softBreakLine if (canBreakAfter(kind) && currentBreakFitWidth <= maxWidth + lineFitEpsilon) { appendWholeSegment(i, w) return finishLine(i + 1, 0, currentBreakPaintWidth) } if (pendingBreakSegmentIndex >= 0 && pendingBreakFitWidth <= maxWidth + lineFitEpsilon) { if ( lineEndSegmentIndex > pendingBreakSegmentIndex || (lineEndSegmentIndex === pendingBreakSegmentIndex && lineEndGraphemeIndex > 0) ) { return finishLine() } return finishLine(pendingBreakSegmentIndex, 0, pendingBreakPaintWidth) } if (w > maxWidth && breakableWidths[i] !== null) { const currentLine = finishLine() if (currentLine !== null) return currentLine const line = appendBreakableSegmentFrom(i, 0) if (line !== null) return line } return finishLine() } appendWholeSegment(i, w) updatePendingBreakForWholeSegment(i, w) } if (pendingBreakSegmentIndex === chunk.consumedEndSegmentIndex && lineEndGraphemeIndex === 0) { return finishLine(chunk.consumedEndSegmentIndex, 0, pendingBreakPaintWidth) } return finishLine(chunk.consumedEndSegmentIndex, 0, lineW) } function layoutNextLineRangeSimple( prepared: PreparedLineBreakData, normalizedStart: LineBreakCursor, maxWidth: number, ): InternalLayoutLine | null { const { widths, kinds, breakableWidths, breakablePrefixWidths } = prepared const engineProfile = getEngineProfile() const lineFitEpsilon = engineProfile.lineFitEpsilon let lineW = 0 let hasContent = false const lineStartSegmentIndex = normalizedStart.segmentIndex const lineStartGraphemeIndex = normalizedStart.graphemeIndex let lineEndSegmentIndex = lineStartSegmentIndex let lineEndGraphemeIndex = lineStartGraphemeIndex let pendingBreakSegmentIndex = -1 let pendingBreakPaintWidth = 0 function finishLine( endSegmentIndex = lineEndSegmentIndex, endGraphemeIndex = lineEndGraphemeIndex, width = lineW, ): InternalLayoutLine | null { if (!hasContent) return null return { startSegmentIndex: lineStartSegmentIndex, startGraphemeIndex: lineStartGraphemeIndex, endSegmentIndex, endGraphemeIndex, width, } } function startLineAtSegment(segmentIndex: number, width: number): void { hasContent = true lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 lineW = width } function startLineAtGrapheme(segmentIndex: number, graphemeIndex: number, width: number): void { hasContent = true lineEndSegmentIndex = segmentIndex lineEndGraphemeIndex = graphemeIndex + 1 lineW = width } function appendWholeSegment(segmentIndex: number, width: number): void { if (!hasContent) { startLineAtSegment(segmentIndex, width) return } lineW += width lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 } function updatePendingBreak(segmentIndex: number, segmentWidth: number): void { if (!canBreakAfter(kinds[segmentIndex]!)) return pendingBreakSegmentIndex = segmentIndex + 1 pendingBreakPaintWidth = lineW - segmentWidth } function appendBreakableSegmentFrom(segmentIndex: number, startGraphemeIndex: number): InternalLayoutLine | null { const gWidths = breakableWidths[segmentIndex]! const gPrefixWidths = breakablePrefixWidths[segmentIndex] ?? null for (let g = startGraphemeIndex; g < gWidths.length; g++) { const gw = getBreakableAdvance( gWidths, gPrefixWidths, g, engineProfile.preferPrefixWidthsForBreakableRuns, ) if (!hasContent) { startLineAtGrapheme(segmentIndex, g, gw) continue } if (lineW + gw > maxWidth + lineFitEpsilon) { return finishLine() } lineW += gw lineEndSegmentIndex = segmentIndex lineEndGraphemeIndex = g + 1 } if (hasContent && lineEndSegmentIndex === segmentIndex && lineEndGraphemeIndex === gWidths.length) { lineEndSegmentIndex = segmentIndex + 1 lineEndGraphemeIndex = 0 } return null } for (let i = normalizedStart.segmentIndex; i < widths.length; i++) { const w = widths[i]! const kind = kinds[i]! const startGraphemeIndex = i === normalizedStart.segmentIndex ? normalizedStart.graphemeIndex : 0 if (!hasContent) { if (startGraphemeIndex > 0) { const line = appendBreakableSegmentFrom(i, startGraphemeIndex) if (line !== null) return line } else if (w > maxWidth && breakableWidths[i] !== null) { const line = appendBreakableSegmentFrom(i, 0) if (line !== null) return line } else { startLineAtSegment(i, w) } updatePendingBreak(i, w) continue } const newW = lineW + w if (newW > maxWidth + lineFitEpsilon) { if (canBreakAfter(kind)) { appendWholeSegment(i, w) return finishLine(i + 1, 0, lineW - w) } if (pendingBreakSegmentIndex >= 0) { if ( lineEndSegmentIndex > pendingBreakSegmentIndex || (lineEndSegmentIndex === pendingBreakSegmentIndex && lineEndGraphemeIndex > 0) ) { return finishLine() } return finishLine(pendingBreakSegmentIndex, 0, pendingBreakPaintWidth) } if (w > maxWidth && breakableWidths[i] !== null) { const currentLine = finishLine() if (currentLine !== null) return currentLine const line = appendBreakableSegmentFrom(i, 0) if (line !== null) return line } return finishLine() } appendWholeSegment(i, w) updatePendingBreak(i, w) } return finishLine() } 9b:T17c5,Command: /bin/zsh -lc "sed -n '1,260p' src/analysis.ts" Chunk ID: dbd9cd Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1481 Output: export type WhiteSpaceMode = 'normal' | 'pre-wrap' export type SegmentBreakKind = | 'text' | 'space' | 'preserved-space' | 'tab' | 'glue' | 'zero-width-break' | 'soft-hyphen' | 'hard-break' type SegmentationPiece = { text: string isWordLike: boolean kind: SegmentBreakKind start: number } export type MergedSegmentation = { len: number texts: string[] isWordLike: boolean[] kinds: SegmentBreakKind[] starts: number[] } export type AnalysisChunk = { startSegmentIndex: number endSegmentIndex: number consumedEndSegmentIndex: number } export type TextAnalysis = { normalized: string, chunks: AnalysisChunk[] } & MergedSegmentation export type AnalysisProfile = { carryCJKAfterClosingQuote: boolean } const collapsibleWhitespaceRunRe = /[ \t\n\r\f]+/g const needsWhitespaceNormalizationRe = /[\t\n\r\f]| {2,}|^ | $/ type WhiteSpaceProfile = { mode: WhiteSpaceMode preserveOrdinarySpaces: boolean preserveHardBreaks: boolean } function getWhiteSpaceProfile(whiteSpace?: WhiteSpaceMode): WhiteSpaceProfile { const mode = whiteSpace ?? 'normal' return mode === 'pre-wrap' ? { mode, preserveOrdinarySpaces: true, preserveHardBreaks: true } : { mode, preserveOrdinarySpaces: false, preserveHardBreaks: false } } export function normalizeWhitespaceNormal(text: string): string { if (!needsWhitespaceNormalizationRe.test(text)) return text let normalized = text.replace(collapsibleWhitespaceRunRe, ' ') if (normalized.charCodeAt(0) === 0x20) { normalized = normalized.slice(1) } if (normalized.length > 0 && normalized.charCodeAt(normalized.length - 1) === 0x20) { normalized = normalized.slice(0, -1) } return normalized } function normalizeWhitespacePreWrap(text: string): string { if (!/[\r\f]/.test(text)) return text.replace(/\r\n/g, '\n') return text .replace(/\r\n/g, '\n') .replace(/[\r\f]/g, '\n') } let sharedWordSegmenter: Intl.Segmenter | null = null let segmenterLocale: string | undefined function getSharedWordSegmenter(): Intl.Segmenter { if (sharedWordSegmenter === null) { sharedWordSegmenter = new Intl.Segmenter(segmenterLocale, { granularity: 'word' }) } return sharedWordSegmenter } export function clearAnalysisCaches(): void { sharedWordSegmenter = null } export function setAnalysisLocale(locale?: string): void { const nextLocale = locale && locale.length > 0 ? locale : undefined if (segmenterLocale === nextLocale) return segmenterLocale = nextLocale sharedWordSegmenter = null } const arabicScriptRe = /\p{Script=Arabic}/u const combiningMarkRe = /\p{M}/u const decimalDigitRe = /\p{Nd}/u function containsArabicScript(text: string): boolean { return arabicScriptRe.test(text) } export function isCJK(s: string): boolean { for (const ch of s) { const c = ch.codePointAt(0)! if ((c >= 0x4E00 && c <= 0x9FFF) || (c >= 0x3400 && c <= 0x4DBF) || (c >= 0x20000 && c <= 0x2A6DF) || (c >= 0x2A700 && c <= 0x2B73F) || (c >= 0x2B740 && c <= 0x2B81F) || (c >= 0x2B820 && c <= 0x2CEAF) || (c >= 0x2CEB0 && c <= 0x2EBEF) || (c >= 0x30000 && c <= 0x3134F) || (c >= 0xF900 && c <= 0xFAFF) || (c >= 0x2F800 && c <= 0x2FA1F) || (c >= 0x3000 && c <= 0x303F) || (c >= 0x3040 && c <= 0x309F) || (c >= 0x30A0 && c <= 0x30FF) || (c >= 0xAC00 && c <= 0xD7AF) || (c >= 0xFF00 && c <= 0xFFEF)) { return true } } return false } export const kinsokuStart = new Set([ '\uFF0C', '\uFF0E', '\uFF01', '\uFF1A', '\uFF1B', '\uFF1F', '\u3001', '\u3002', '\u30FB', '\uFF09', '\u3015', '\u3009', '\u300B', '\u300D', '\u300F', '\u3011', '\u3017', '\u3019', '\u301B', '\u30FC', '\u3005', '\u303B', '\u309D', '\u309E', '\u30FD', '\u30FE', ]) export const kinsokuEnd = new Set([ '"', '(', '[', '{', '“', '‘', '«', '‹', '\uFF08', '\u3014', '\u3008', '\u300A', '\u300C', '\u300E', '\u3010', '\u3016', '\u3018', '\u301A', ]) const forwardStickyGlue = new Set([ "'", '’', ]) export const leftStickyPunctuation = new Set([ '.', ',', '!', '?', ':', ';', '\u060C', '\u061B', '\u061F', '\u0964', '\u0965', '\u104A', '\u104B', '\u104C', '\u104D', '\u104F', ')', ']', '}', '%', '"', '”', '’', '»', '›', '…', ]) const arabicNoSpaceTrailingPunctuation = new Set([ ':', '.', '\u060C', '\u061B', ]) const myanmarMedialGlue = new Set([ '\u104F', ]) const closingQuoteChars = new Set([ '”', '’', '»', '›', '\u300D', '\u300F', '\u3011', '\u300B', '\u3009', '\u3015', '\uFF09', ]) function isLeftStickyPunctuationSegment(segment: string): boolean { if (isEscapedQuoteClusterSegment(segment)) return true let sawPunctuation = false for (const ch of segment) { if (leftStickyPunctuation.has(ch)) { sawPunctuation = true continue } if (sawPunctuation && combiningMarkRe.test(ch)) continue return false } return sawPunctuation } function isCJKLineStartProhibitedSegment(segment: string): boolean { for (const ch of segment) { if (!kinsokuStart.has(ch) && !leftStickyPunctuation.has(ch)) return false } return segment.length > 0 } function isForwardStickyClusterSegment(segment: string): boolean { if (isEscapedQuoteClusterSegment(segment)) return true for (const ch of segment) { if (!kinsokuEnd.has(ch) && !forwardStickyGlue.has(ch) && !combiningMarkRe.test(ch)) return false } return segment.length > 0 } function isEscapedQuoteClusterSegment(segment: string): boolean { let sawQuote = false for (const ch of segment) { if (ch === '\\' || combiningMarkRe.test(ch)) continue if (kinsokuEnd.has(ch) || leftStickyPunctuation.has(ch) || forwardStickyGlue.has(ch)) { sawQuote = true continue } return false } return sawQuote } 9c:T1bea,Command: /bin/zsh -lc "sed -n '261,520p' src/analysis.ts" Chunk ID: f7a014 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1746 Output: function splitTrailingForwardStickyCluster(text: string): { head: string, tail: string } | null { const chars = Array.from(text) let splitIndex = chars.length while (splitIndex > 0) { const ch = chars[splitIndex - 1]! if (combiningMarkRe.test(ch)) { splitIndex-- continue } if (kinsokuEnd.has(ch) || forwardStickyGlue.has(ch)) { splitIndex-- continue } break } if (splitIndex <= 0 || splitIndex === chars.length) return null return { head: chars.slice(0, splitIndex).join(''), tail: chars.slice(splitIndex).join(''), } } function isRepeatedSingleCharRun(segment: string, ch: string): boolean { if (segment.length === 0) return false for (const part of segment) { if (part !== ch) return false } return true } function endsWithArabicNoSpacePunctuation(segment: string): boolean { if (!containsArabicScript(segment) || segment.length === 0) return false return arabicNoSpaceTrailingPunctuation.has(segment[segment.length - 1]!) } function endsWithMyanmarMedialGlue(segment: string): boolean { if (segment.length === 0) return false return myanmarMedialGlue.has(segment[segment.length - 1]!) } function splitLeadingSpaceAndMarks(segment: string): { space: string, marks: string } | null { if (segment.length < 2 || segment[0] !== ' ') return null const marks = segment.slice(1) if (/^\p{M}+$/u.test(marks)) { return { space: ' ', marks } } return null } export function endsWithClosingQuote(text: string): boolean { for (let i = text.length - 1; i >= 0; i--) { const ch = text[i]! if (closingQuoteChars.has(ch)) return true if (!leftStickyPunctuation.has(ch)) return false } return false } function classifySegmentBreakChar(ch: string, whiteSpaceProfile: WhiteSpaceProfile): SegmentBreakKind { if (whiteSpaceProfile.preserveOrdinarySpaces || whiteSpaceProfile.preserveHardBreaks) { if (ch === ' ') return 'preserved-space' if (ch === '\t') return 'tab' if (whiteSpaceProfile.preserveHardBreaks && ch === '\n') return 'hard-break' } if (ch === ' ') return 'space' if (ch === '\u00A0' || ch === '\u202F' || ch === '\u2060' || ch === '\uFEFF') { return 'glue' } if (ch === '\u200B') return 'zero-width-break' if (ch === '\u00AD') return 'soft-hyphen' return 'text' } function splitSegmentByBreakKind( segment: string, isWordLike: boolean, start: number, whiteSpaceProfile: WhiteSpaceProfile, ): SegmentationPiece[] { const pieces: SegmentationPiece[] = [] let currentKind: SegmentBreakKind | null = null let currentText = '' let currentStart = start let currentWordLike = false let offset = 0 for (const ch of segment) { const kind = classifySegmentBreakChar(ch, whiteSpaceProfile) const wordLike = kind === 'text' && isWordLike if (currentKind !== null && kind === currentKind && wordLike === currentWordLike) { currentText += ch offset += ch.length continue } if (currentKind !== null) { pieces.push({ text: currentText, isWordLike: currentWordLike, kind: currentKind, start: currentStart, }) } currentKind = kind currentText = ch currentStart = start + offset currentWordLike = wordLike offset += ch.length } if (currentKind !== null) { pieces.push({ text: currentText, isWordLike: currentWordLike, kind: currentKind, start: currentStart, }) } return pieces } function isTextRunBoundary(kind: SegmentBreakKind): boolean { return ( kind === 'space' || kind === 'preserved-space' || kind === 'zero-width-break' || kind === 'hard-break' ) } const urlSchemeSegmentRe = /^[A-Za-z][A-Za-z0-9+.-]*:$/ function isUrlLikeRunStart(segmentation: MergedSegmentation, index: number): boolean { const text = segmentation.texts[index]! if (text.startsWith('www.')) return true return ( urlSchemeSegmentRe.test(text) && index + 1 < segmentation.len && segmentation.kinds[index + 1] === 'text' && segmentation.texts[index + 1] === '//' ) } function isUrlQueryBoundarySegment(text: string): boolean { return text.includes('?') && (text.includes('://') || text.startsWith('www.')) } function mergeUrlLikeRuns(segmentation: MergedSegmentation): MergedSegmentation { const texts = segmentation.texts.slice() const isWordLike = segmentation.isWordLike.slice() const kinds = segmentation.kinds.slice() const starts = segmentation.starts.slice() for (let i = 0; i < segmentation.len; i++) { if (kinds[i] !== 'text' || !isUrlLikeRunStart(segmentation, i)) continue let j = i + 1 while (j < segmentation.len && !isTextRunBoundary(kinds[j]!)) { texts[i] += texts[j]! isWordLike[i] = true const endsQueryPrefix = texts[j]!.includes('?') kinds[j] = 'text' texts[j] = '' j++ if (endsQueryPrefix) break } } let compactLen = 0 for (let read = 0; read < texts.length; read++) { const text = texts[read]! if (text.length === 0) continue if (compactLen !== read) { texts[compactLen] = text isWordLike[compactLen] = isWordLike[read]! kinds[compactLen] = kinds[read]! starts[compactLen] = starts[read]! } compactLen++ } texts.length = compactLen isWordLike.length = compactLen kinds.length = compactLen starts.length = compactLen return { len: compactLen, texts, isWordLike, kinds, starts, } } function mergeUrlQueryRuns(segmentation: MergedSegmentation): MergedSegmentation { const texts: string[] = [] const isWordLike: boolean[] = [] const kinds: SegmentBreakKind[] = [] const starts: number[] = [] for (let i = 0; i < segmentation.len; i++) { const text = segmentation.texts[i]! texts.push(text) isWordLike.push(segmentation.isWordLike[i]!) kinds.push(segmentation.kinds[i]!) starts.push(segmentation.starts[i]!) if (!isUrlQueryBoundarySegment(text)) continue const nextIndex = i + 1 if ( nextIndex >= segmentation.len || isTextRunBoundary(segmentation.kinds[nextIndex]!) ) { continue } let queryText = '' const queryStart = segmentation.starts[nextIndex]! let j = nextIndex while (j < segmentation.len && !isTextRunBoundary(segmentation.kinds[j]!)) { queryText += segmentation.texts[j]! j++ } if (queryText.length > 0) { texts.push(queryText) isWordLike.push(true) kinds.push('text') starts.push(queryStart) i = j - 1 } } return { len: texts.length, texts, isWordLike, kinds, starts, } } const numericJoinerChars = new Set([ ':', '-', '/', '×', ',', '.', '+', '\u2013', '\u2014', ]) const asciiPunctuationChainSegmentRe = /^[A-Za-z0-9_]+[,:;]*$/ const asciiPunctuationChainTrailingJoinersRe = /[,:;]+$/ function segmentContainsDecimalDigit(text: string): boolean { for (const ch of text) { 9d:T1afb,Command: /bin/zsh -lc "sed -n '521,780p' src/analysis.ts" Chunk ID: ee2603 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1686 Output: if (decimalDigitRe.test(ch)) return true } return false } function isNumericRunSegment(text: string): boolean { if (text.length === 0) return false for (const ch of text) { if (decimalDigitRe.test(ch) || numericJoinerChars.has(ch)) continue return false } return true } function mergeNumericRuns(segmentation: MergedSegmentation): MergedSegmentation { const texts: string[] = [] const isWordLike: boolean[] = [] const kinds: SegmentBreakKind[] = [] const starts: number[] = [] for (let i = 0; i < segmentation.len; i++) { const text = segmentation.texts[i]! const kind = segmentation.kinds[i]! if (kind === 'text' && isNumericRunSegment(text) && segmentContainsDecimalDigit(text)) { let mergedText = text let j = i + 1 while ( j < segmentation.len && segmentation.kinds[j] === 'text' && isNumericRunSegment(segmentation.texts[j]!) ) { mergedText += segmentation.texts[j]! j++ } texts.push(mergedText) isWordLike.push(true) kinds.push('text') starts.push(segmentation.starts[i]!) i = j - 1 continue } texts.push(text) isWordLike.push(segmentation.isWordLike[i]!) kinds.push(kind) starts.push(segmentation.starts[i]!) } return { len: texts.length, texts, isWordLike, kinds, starts, } } function mergeAsciiPunctuationChains(segmentation: MergedSegmentation): MergedSegmentation { const texts: string[] = [] const isWordLike: boolean[] = [] const kinds: SegmentBreakKind[] = [] const starts: number[] = [] for (let i = 0; i < segmentation.len; i++) { const text = segmentation.texts[i]! const kind = segmentation.kinds[i]! const wordLike = segmentation.isWordLike[i]! if (kind === 'text' && wordLike && asciiPunctuationChainSegmentRe.test(text)) { let mergedText = text let j = i + 1 while ( asciiPunctuationChainTrailingJoinersRe.test(mergedText) && j < segmentation.len && segmentation.kinds[j] === 'text' && segmentation.isWordLike[j] && asciiPunctuationChainSegmentRe.test(segmentation.texts[j]!) ) { mergedText += segmentation.texts[j]! j++ } texts.push(mergedText) isWordLike.push(true) kinds.push('text') starts.push(segmentation.starts[i]!) i = j - 1 continue } texts.push(text) isWordLike.push(wordLike) kinds.push(kind) starts.push(segmentation.starts[i]!) } return { len: texts.length, texts, isWordLike, kinds, starts, } } function splitHyphenatedNumericRuns(segmentation: MergedSegmentation): MergedSegmentation { const texts: string[] = [] const isWordLike: boolean[] = [] const kinds: SegmentBreakKind[] = [] const starts: number[] = [] for (let i = 0; i < segmentation.len; i++) { const text = segmentation.texts[i]! if (segmentation.kinds[i] === 'text' && text.includes('-')) { const parts = text.split('-') let shouldSplit = parts.length > 1 for (let j = 0; j < parts.length; j++) { const part = parts[j]! if (!shouldSplit) break if ( part.length === 0 || !segmentContainsDecimalDigit(part) || !isNumericRunSegment(part) ) { shouldSplit = false } } if (shouldSplit) { let offset = 0 for (let j = 0; j < parts.length; j++) { const part = parts[j]! const splitText = j < parts.length - 1 ? `${part}-` : part texts.push(splitText) isWordLike.push(true) kinds.push('text') starts.push(segmentation.starts[i]! + offset) offset += splitText.length } continue } } texts.push(text) isWordLike.push(segmentation.isWordLike[i]!) kinds.push(segmentation.kinds[i]!) starts.push(segmentation.starts[i]!) } return { len: texts.length, texts, isWordLike, kinds, starts, } } function mergeGlueConnectedTextRuns(segmentation: MergedSegmentation): MergedSegmentation { const texts: string[] = [] const isWordLike: boolean[] = [] const kinds: SegmentBreakKind[] = [] const starts: number[] = [] let read = 0 while (read < segmentation.len) { let text = segmentation.texts[read]! let wordLike = segmentation.isWordLike[read]! let kind = segmentation.kinds[read]! let start = segmentation.starts[read]! if (kind === 'glue') { let glueText = text const glueStart = start read++ while (read < segmentation.len && segmentation.kinds[read] === 'glue') { glueText += segmentation.texts[read]! read++ } if (read < segmentation.len && segmentation.kinds[read] === 'text') { text = glueText + segmentation.texts[read]! wordLike = segmentation.isWordLike[read]! kind = 'text' start = glueStart read++ } else { texts.push(glueText) isWordLike.push(false) kinds.push('glue') starts.push(glueStart) continue } } else { read++ } if (kind === 'text') { while (read < segmentation.len && segmentation.kinds[read] === 'glue') { let glueText = '' while (read < segmentation.len && segmentation.kinds[read] === 'glue') { glueText += segmentation.texts[read]! read++ } if (read < segmentation.len && segmentation.kinds[read] === 'text') { text += glueText + segmentation.texts[read]! wordLike = wordLike || segmentation.isWordLike[read]! read++ continue } text += glueText } } texts.push(text) isWordLike.push(wordLike) kinds.push(kind) starts.push(start) } return { len: texts.length, texts, isWordLike, kinds, starts, } } function carryTrailingForwardStickyAcrossCJKBoundary(segmentation: MergedSegmentation): MergedSegmentation { const texts = segmentation.texts.slice() const isWordLike = segmentation.isWordLike.slice() const kinds = segmentation.kinds.slice() const starts = segmentation.starts.slice() for (let i = 0; i < texts.length - 1; i++) { if (kinds[i] !== 'text' || kinds[i + 1] !== 'text') continue if (!isCJK(texts[i]!) || !isCJK(texts[i + 1]!)) continue const split = splitTrailingForwardStickyCluster(texts[i]!) if (split === null) continue texts[i] = split.head texts[i + 1] = split.tail + texts[i + 1]! starts[i + 1] = starts[i]! + split.head.length } return { len: texts.length, texts, isWordLike, kinds, starts, } } 9e:T1c76,Command: /bin/zsh -lc "sed -n '781,1040p' src/analysis.ts" Chunk ID: 3623ef Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1781 Output: function buildMergedSegmentation( normalized: string, profile: AnalysisProfile, whiteSpaceProfile: WhiteSpaceProfile, ): MergedSegmentation { const wordSegmenter = getSharedWordSegmenter() let mergedLen = 0 const mergedTexts: string[] = [] const mergedWordLike: boolean[] = [] const mergedKinds: SegmentBreakKind[] = [] const mergedStarts: number[] = [] for (const s of wordSegmenter.segment(normalized)) { for (const piece of splitSegmentByBreakKind(s.segment, s.isWordLike ?? false, s.index, whiteSpaceProfile)) { const isText = piece.kind === 'text' if ( profile.carryCJKAfterClosingQuote && isText && mergedLen > 0 && mergedKinds[mergedLen - 1] === 'text' && isCJK(piece.text) && isCJK(mergedTexts[mergedLen - 1]!) && endsWithClosingQuote(mergedTexts[mergedLen - 1]!) ) { mergedTexts[mergedLen - 1] += piece.text mergedWordLike[mergedLen - 1] = mergedWordLike[mergedLen - 1]! || piece.isWordLike } else if ( isText && mergedLen > 0 && mergedKinds[mergedLen - 1] === 'text' && isCJKLineStartProhibitedSegment(piece.text) && isCJK(mergedTexts[mergedLen - 1]!) ) { mergedTexts[mergedLen - 1] += piece.text mergedWordLike[mergedLen - 1] = mergedWordLike[mergedLen - 1]! || piece.isWordLike } else if ( isText && mergedLen > 0 && mergedKinds[mergedLen - 1] === 'text' && endsWithMyanmarMedialGlue(mergedTexts[mergedLen - 1]!) ) { mergedTexts[mergedLen - 1] += piece.text mergedWordLike[mergedLen - 1] = mergedWordLike[mergedLen - 1]! || piece.isWordLike } else if ( isText && mergedLen > 0 && mergedKinds[mergedLen - 1] === 'text' && piece.isWordLike && containsArabicScript(piece.text) && endsWithArabicNoSpacePunctuation(mergedTexts[mergedLen - 1]!) ) { mergedTexts[mergedLen - 1] += piece.text mergedWordLike[mergedLen - 1] = true } else if ( isText && !piece.isWordLike && mergedLen > 0 && mergedKinds[mergedLen - 1] === 'text' && piece.text.length === 1 && piece.text !== '-' && piece.text !== '—' && isRepeatedSingleCharRun(mergedTexts[mergedLen - 1]!, piece.text) ) { mergedTexts[mergedLen - 1] += piece.text } else if ( isText && !piece.isWordLike && mergedLen > 0 && mergedKinds[mergedLen - 1] === 'text' && ( isLeftStickyPunctuationSegment(piece.text) || (piece.text === '-' && mergedWordLike[mergedLen - 1]!) ) ) { mergedTexts[mergedLen - 1] += piece.text } else { mergedTexts[mergedLen] = piece.text mergedWordLike[mergedLen] = piece.isWordLike mergedKinds[mergedLen] = piece.kind mergedStarts[mergedLen] = piece.start mergedLen++ } } } for (let i = 1; i < mergedLen; i++) { if ( mergedKinds[i] === 'text' && !mergedWordLike[i]! && isEscapedQuoteClusterSegment(mergedTexts[i]!) && mergedKinds[i - 1] === 'text' ) { mergedTexts[i - 1] += mergedTexts[i]! mergedWordLike[i - 1] = mergedWordLike[i - 1]! || mergedWordLike[i]! mergedTexts[i] = '' } } for (let i = mergedLen - 2; i >= 0; i--) { if (mergedKinds[i] === 'text' && !mergedWordLike[i]! && isForwardStickyClusterSegment(mergedTexts[i]!)) { let j = i + 1 while (j < mergedLen && mergedTexts[j] === '') j++ if (j < mergedLen && mergedKinds[j] === 'text') { mergedTexts[j] = mergedTexts[i]! + mergedTexts[j]! mergedStarts[j] = mergedStarts[i]! mergedTexts[i] = '' } } } let compactLen = 0 for (let read = 0; read < mergedLen; read++) { const text = mergedTexts[read]! if (text.length === 0) continue if (compactLen !== read) { mergedTexts[compactLen] = text mergedWordLike[compactLen] = mergedWordLike[read]! mergedKinds[compactLen] = mergedKinds[read]! mergedStarts[compactLen] = mergedStarts[read]! } compactLen++ } mergedTexts.length = compactLen mergedWordLike.length = compactLen mergedKinds.length = compactLen mergedStarts.length = compactLen const compacted = mergeGlueConnectedTextRuns({ len: compactLen, texts: mergedTexts, isWordLike: mergedWordLike, kinds: mergedKinds, starts: mergedStarts, }) const withMergedUrls = carryTrailingForwardStickyAcrossCJKBoundary( mergeAsciiPunctuationChains( splitHyphenatedNumericRuns(mergeNumericRuns(mergeUrlQueryRuns(mergeUrlLikeRuns(compacted)))), ), ) for (let i = 0; i < withMergedUrls.len - 1; i++) { const split = splitLeadingSpaceAndMarks(withMergedUrls.texts[i]!) if (split === null) continue if ( (withMergedUrls.kinds[i] !== 'space' && withMergedUrls.kinds[i] !== 'preserved-space') || withMergedUrls.kinds[i + 1] !== 'text' || !containsArabicScript(withMergedUrls.texts[i + 1]!) ) { continue } withMergedUrls.texts[i] = split.space withMergedUrls.isWordLike[i] = false withMergedUrls.kinds[i] = withMergedUrls.kinds[i] === 'preserved-space' ? 'preserved-space' : 'space' withMergedUrls.texts[i + 1] = split.marks + withMergedUrls.texts[i + 1]! withMergedUrls.starts[i + 1] = withMergedUrls.starts[i]! + split.space.length } return withMergedUrls } function compileAnalysisChunks(segmentation: MergedSegmentation, whiteSpaceProfile: WhiteSpaceProfile): AnalysisChunk[] { if (segmentation.len === 0) return [] if (!whiteSpaceProfile.preserveHardBreaks) { return [{ startSegmentIndex: 0, endSegmentIndex: segmentation.len, consumedEndSegmentIndex: segmentation.len, }] } const chunks: AnalysisChunk[] = [] let startSegmentIndex = 0 for (let i = 0; i < segmentation.len; i++) { if (segmentation.kinds[i] !== 'hard-break') continue chunks.push({ startSegmentIndex, endSegmentIndex: i, consumedEndSegmentIndex: i + 1, }) startSegmentIndex = i + 1 } if (startSegmentIndex < segmentation.len) { chunks.push({ startSegmentIndex, endSegmentIndex: segmentation.len, consumedEndSegmentIndex: segmentation.len, }) } return chunks } export function analyzeText( text: string, profile: AnalysisProfile, whiteSpace: WhiteSpaceMode = 'normal', ): TextAnalysis { const whiteSpaceProfile = getWhiteSpaceProfile(whiteSpace) const normalized = whiteSpaceProfile.mode === 'pre-wrap' ? normalizeWhitespacePreWrap(text) : normalizeWhitespaceNormal(text) if (normalized.length === 0) { return { normalized, chunks: [], len: 0, texts: [], isWordLike: [], kinds: [], starts: [], } } const segmentation = buildMergedSegmentation(normalized, profile, whiteSpaceProfile) return { normalized, chunks: compileAnalysisChunks(segmentation, whiteSpaceProfile), ...segmentation, } } 9f:T2281,Command: /bin/zsh -lc "sed -n '1,260p' src/layout.test.ts" Chunk ID: 137b2e Wall time: 0.0000 seconds Process exited with code 0 Original token count: 2168 Output: import { beforeAll, beforeEach, describe, expect, test } from 'bun:test' // Keep the permanent suite small and durable. These tests exercise the shipped // prepare/layout exports with a deterministic fake canvas backend. For narrow // browser-specific investigations, prefer throwaway probes and browser checkers // over mirroring the full implementation here. const FONT = '16px Test Sans' const LINE_HEIGHT = 19 type LayoutModule = typeof import('./layout.ts') type LineBreakModule = typeof import('./line-break.ts') let prepare: LayoutModule['prepare'] let prepareWithSegments: LayoutModule['prepareWithSegments'] let layout: LayoutModule['layout'] let layoutWithLines: LayoutModule['layoutWithLines'] let layoutNextLine: LayoutModule['layoutNextLine'] let walkLineRanges: LayoutModule['walkLineRanges'] let clearCache: LayoutModule['clearCache'] let setLocale: LayoutModule['setLocale'] let countPreparedLines: LineBreakModule['countPreparedLines'] let walkPreparedLines: LineBreakModule['walkPreparedLines'] const emojiPresentationRe = /\p{Emoji_Presentation}/u const punctuationRe = /[.,!?;:%)\]}'"”’»›…—-]/u const graphemeSegmenter = new Intl.Segmenter(undefined, { granularity: 'grapheme' }) type TestLayoutCursor = { segmentIndex: number graphemeIndex: number } type TestPreparedTextWithSegments = { segments: string[] } type TestLayoutLine = { start: TestLayoutCursor end: TestLayoutCursor } function parseFontSize(font: string): number { const match = font.match(/(\d+(?:\.\d+)?)\s*px/) return match ? Number.parseFloat(match[1]!) : 16 } function isWideCharacter(ch: string): boolean { const code = ch.codePointAt(0)! return ( (code >= 0x4E00 && code <= 0x9FFF) || (code >= 0x3400 && code <= 0x4DBF) || (code >= 0xF900 && code <= 0xFAFF) || (code >= 0x2F800 && code <= 0x2FA1F) || (code >= 0x20000 && code <= 0x2A6DF) || (code >= 0x2A700 && code <= 0x2B73F) || (code >= 0x2B740 && code <= 0x2B81F) || (code >= 0x2B820 && code <= 0x2CEAF) || (code >= 0x2CEB0 && code <= 0x2EBEF) || (code >= 0x30000 && code <= 0x3134F) || (code >= 0x3000 && code <= 0x303F) || (code >= 0x3040 && code <= 0x309F) || (code >= 0x30A0 && code <= 0x30FF) || (code >= 0xAC00 && code <= 0xD7AF) || (code >= 0xFF00 && code <= 0xFFEF) ) } function measureWidth(text: string, font: string): number { const fontSize = parseFontSize(font) let width = 0 for (const ch of text) { if (ch === ' ') { width += fontSize * 0.33 } else if (ch === '\t') { width += fontSize * 1.32 } else if (emojiPresentationRe.test(ch) || ch === '\uFE0F') { width += fontSize } else if (isWideCharacter(ch)) { width += fontSize } else if (punctuationRe.test(ch)) { width += fontSize * 0.4 } else { width += fontSize * 0.6 } } return width } function nextTabAdvance(lineWidth: number, spaceWidth: number, tabSize = 8): number { const tabStopAdvance = spaceWidth * tabSize const remainder = lineWidth % tabStopAdvance return remainder === 0 ? tabStopAdvance : tabStopAdvance - remainder } function getSegmentGraphemes(text: string): string[] { return Array.from(graphemeSegmenter.segment(text), segment => segment.segment) } function slicePreparedText( prepared: TestPreparedTextWithSegments, start: TestLayoutCursor, end: TestLayoutCursor, ): string { if (start.segmentIndex === end.segmentIndex) { const segment = prepared.segments[start.segmentIndex] if (segment === undefined) return '' return getSegmentGraphemes(segment).slice(start.graphemeIndex, end.graphemeIndex).join('') } let result = '' for (let segmentIndex = start.segmentIndex; segmentIndex < end.segmentIndex; segmentIndex++) { const segment = prepared.segments[segmentIndex] if (segment === undefined) break if (segmentIndex === start.segmentIndex && start.graphemeIndex > 0) { result += getSegmentGraphemes(segment).slice(start.graphemeIndex).join('') } else { result += segment } } if (end.graphemeIndex > 0) { const segment = prepared.segments[end.segmentIndex] if (segment !== undefined) { result += getSegmentGraphemes(segment).slice(0, end.graphemeIndex).join('') } } return result } function reconstructFromLineBoundaries( prepared: TestPreparedTextWithSegments, lines: TestLayoutLine[], ): string { return lines.map(line => slicePreparedText(prepared, line.start, line.end)).join('') } function collectStreamedLines( prepared: TestPreparedTextWithSegments, width: number, ): TestLayoutLine[] { const lines: TestLayoutLine[] = [] let cursor = { segmentIndex: 0, graphemeIndex: 0 } while (true) { const line = layoutNextLine(prepared as Parameters[0], cursor, width) if (line === null) break lines.push(line) cursor = line.end } return lines } function reconstructFromWalkedRanges( prepared: TestPreparedTextWithSegments, width: number, ): string { const slices: string[] = [] walkLineRanges(prepared as Parameters[0], width, line => { slices.push(slicePreparedText(prepared, line.start, line.end)) }) return slices.join('') } class TestCanvasRenderingContext2D { font = '' measureText(text: string): { width: number } { return { width: measureWidth(text, this.font) } } } class TestOffscreenCanvas { constructor(_width: number, _height: number) {} getContext(_kind: string): TestCanvasRenderingContext2D { return new TestCanvasRenderingContext2D() } } beforeAll(async () => { Reflect.set(globalThis, 'OffscreenCanvas', TestOffscreenCanvas) const mod = await import('./layout.ts') const lineBreakMod = await import('./line-break.ts') ;({ prepare, prepareWithSegments, layout, layoutWithLines, layoutNextLine, walkLineRanges, clearCache, setLocale, } = mod) ;({ countPreparedLines, walkPreparedLines } = lineBreakMod) }) beforeEach(() => { setLocale(undefined) clearCache() }) describe('prepare invariants', () => { test('whitespace-only input stays empty', () => { const prepared = prepare(' \t\n ', FONT) expect(layout(prepared, 200, LINE_HEIGHT)).toEqual({ lineCount: 0, height: 0 }) }) test('collapses ordinary whitespace runs and trims the edges', () => { const prepared = prepareWithSegments(' Hello\t \n World ', FONT) expect(prepared.segments).toEqual(['Hello', ' ', 'World']) }) test('pre-wrap mode keeps ordinary spaces instead of collapsing them', () => { const prepared = prepareWithSegments(' Hello World ', FONT, { whiteSpace: 'pre-wrap' }) expect(prepared.segments).toEqual([' ', 'Hello', ' ', 'World', ' ']) expect(prepared.kinds).toEqual(['preserved-space', 'text', 'preserved-space', 'text', 'preserved-space']) }) test('pre-wrap mode keeps hard breaks as explicit segments', () => { const prepared = prepareWithSegments('Hello\nWorld', FONT, { whiteSpace: 'pre-wrap' }) expect(prepared.segments).toEqual(['Hello', '\n', 'World']) expect(prepared.kinds).toEqual(['text', 'hard-break', 'text']) }) test('pre-wrap mode normalizes CRLF into a single hard break', () => { const prepared = prepareWithSegments('Hello\r\nWorld', FONT, { whiteSpace: 'pre-wrap' }) expect(prepared.segments).toEqual(['Hello', '\n', 'World']) expect(prepared.kinds).toEqual(['text', 'hard-break', 'text']) }) test('pre-wrap mode keeps tabs as explicit segments', () => { const prepared = prepareWithSegments('Hello\tWorld', FONT, { whiteSpace: 'pre-wrap' }) expect(prepared.segments).toEqual(['Hello', '\t', 'World']) expect(prepared.kinds).toEqual(['text', 'tab', 'text']) }) test('keeps non-breaking spaces as glue instead of collapsing them away', () => { const prepared = prepareWithSegments('Hello\u00A0world', FONT) expect(prepared.segments).toEqual(['Hello\u00A0world']) expect(prepared.kinds).toEqual(['text']) }) test('keeps standalone non-breaking spaces as visible glue content', () => { const prepared = prepareWithSegments('\u00A0', FONT) expect(prepared.segments).toEqual(['\u00A0']) expect(layout(prepared, 200, LINE_HEIGHT)).toEqual({ lineCount: 1, height: LINE_HEIGHT }) }) test('pre-wrap mode keeps whitespace-only input visible', () => { const prepared = prepare(' ', FONT, { whiteSpace: 'pre-wrap' }) expect(layout(prepared, 200, LINE_HEIGHT)).toEqual({ lineCount: 1, height: LINE_HEIGHT }) }) test('keeps narrow no-break spaces as glue content', () => { const prepared = prepareWithSegments('10\u202F000', FONT) a0:T2db2,Command: /bin/zsh -lc "sed -n '261,520p' src/layout.test.ts" Chunk ID: 06fed2 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 2883 Output: expect(prepared.segments).toEqual(['10\u202F000']) expect(prepared.kinds).toEqual(['text']) }) test('keeps word joiners as glue content', () => { const prepared = prepareWithSegments('foo\u2060bar', FONT) expect(prepared.segments).toEqual(['foo\u2060bar']) expect(prepared.kinds).toEqual(['text']) }) test('treats zero-width spaces as explicit break opportunities', () => { const prepared = prepareWithSegments('alpha\u200Bbeta', FONT) expect(prepared.segments).toEqual(['alpha', '\u200B', 'beta']) expect(prepared.kinds).toEqual(['text', 'zero-width-break', 'text']) const alphaWidth = prepared.widths[0]! expect(layout(prepared, alphaWidth + 0.1, LINE_HEIGHT).lineCount).toBe(2) }) test('treats soft hyphens as discretionary break points', () => { const prepared = prepareWithSegments('trans\u00ADatlantic', FONT) expect(prepared.segments).toEqual(['trans', '\u00AD', 'atlantic']) expect(prepared.kinds).toEqual(['text', 'soft-hyphen', 'text']) const wide = layoutWithLines(prepared, 200, LINE_HEIGHT) expect(wide.lineCount).toBe(1) expect(wide.lines.map(line => line.text)).toEqual(['transatlantic']) const prefixed = prepareWithSegments('foo trans\u00ADatlantic', FONT) const softBreakWidth = Math.max( prefixed.widths[0]! + prefixed.widths[1]! + prefixed.widths[2]! + prefixed.discretionaryHyphenWidth, prefixed.widths[4]!, ) + 0.1 const narrow = layoutWithLines(prefixed, softBreakWidth, LINE_HEIGHT) expect(narrow.lineCount).toBe(2) expect(narrow.lines.map(line => line.text)).toEqual(['foo trans-', 'atlantic']) expect(layout(prefixed, softBreakWidth, LINE_HEIGHT).lineCount).toBe(narrow.lineCount) const continuedSoftBreakWidth = prefixed.widths[0]! + prefixed.widths[1]! + prefixed.widths[2]! + prefixed.breakableWidths[4]![0]! + prefixed.discretionaryHyphenWidth + 0.1 const continued = layoutWithLines(prefixed, continuedSoftBreakWidth, LINE_HEIGHT) expect(continued.lines.map(line => line.text)).toEqual(['foo trans-a', 'tlantic']) expect(layout(prefixed, continuedSoftBreakWidth, LINE_HEIGHT).lineCount).toBe(continued.lineCount) }) test('keeps closing punctuation attached to the preceding word', () => { const prepared = prepareWithSegments('hello.', FONT) expect(prepared.segments).toEqual(['hello.']) }) test('keeps arabic punctuation attached to the preceding word', () => { const prepared = prepareWithSegments('مرحبا، عالم؟', FONT) expect(prepared.segments).toEqual(['مرحبا،', ' ', 'عالم؟']) }) test('keeps arabic punctuation-plus-mark clusters attached to the preceding word', () => { const prepared = prepareWithSegments('وحوارى بكشء،ٍ من قولهم', FONT) expect(prepared.segments).toEqual(['وحوارى', ' ', 'بكشء،ٍ', ' ', 'من', ' ', 'قولهم']) }) test('keeps arabic no-space punctuation clusters together', () => { const prepared = prepareWithSegments('فيقول:وعليك السلام', FONT) expect(prepared.segments).toEqual(['فيقول:وعليك', ' ', 'السلام']) }) test('keeps arabic comma-followed text together without a space', () => { const prepared = prepareWithSegments('همزةٌ،ما كان', FONT) expect(prepared.segments).toEqual(['همزةٌ،ما', ' ', 'كان']) }) test('keeps leading arabic combining marks with the following word', () => { const prepared = prepareWithSegments('كل ِّواحدةٍ', FONT) expect(prepared.segments).toEqual(['كل', ' ', 'ِّواحدةٍ']) }) test('keeps devanagari danda punctuation attached to the preceding word', () => { const prepared = prepareWithSegments('नमस्ते। दुनिया॥', FONT) expect(prepared.segments).toEqual(['नमस्ते।', ' ', 'दुनिया॥']) }) test('keeps myanmar punctuation attached to the preceding word', () => { const prepared = prepareWithSegments('ဖြစ်သည်။ နောက်တစ်ခု၊ ကိုက်ချီ၍ ယုံကြည်မိကြ၏။', FONT) expect(prepared.segments.slice(0, 7)).toEqual(['ဖြစ်သည်။', ' ', 'နောက်တစ်ခု၊', ' ', 'ကိုက်', 'ချီ၍', ' ']) expect(prepared.segments.at(-1)).toBe('ကြ၏။') }) test('keeps myanmar possessive marker attached to the following word', () => { const prepared = prepareWithSegments('ကျွန်ုပ်၏လက်မဖြင့်', FONT) expect(prepared.segments).toEqual(['ကျွန်ုပ်၏လက်မ', 'ဖြင့်']) }) test('keeps opening quotes attached to the following word', () => { const prepared = prepareWithSegments('“Whenever', FONT) expect(prepared.segments).toEqual(['“Whenever']) }) test('keeps apostrophe-led elisions attached to the following word', () => { const prepared = prepareWithSegments('“Take ’em downstairs', FONT) expect(prepared.segments).toEqual(['“Take', ' ', '’em', ' ', 'downstairs']) }) test('keeps stacked opening quotes attached to the following word', () => { const prepared = prepareWithSegments('invented, “‘George B. Wilson', FONT) expect(prepared.segments).toEqual(['invented,', ' ', '“‘George', ' ', 'B.', ' ', 'Wilson']) }) test('treats ascii quotes as opening and closing glue by context', () => { const prepared = prepareWithSegments('said "hello" there', FONT) expect(prepared.segments).toEqual(['said', ' ', '"hello"', ' ', 'there']) }) test('treats escaped ascii quote clusters as opening and closing glue by context', () => { const text = String.raw`say \"hello\" there` const prepared = prepareWithSegments(text, FONT) expect(prepared.segments).toEqual(['say', ' ', String.raw`\"hello\"`, ' ', 'there']) }) test('keeps URL-like runs together as one breakable segment', () => { const prepared = prepareWithSegments('see https://example.com/reports/q3?lang=ar&mode=full now', FONT) expect(prepared.segments).toEqual([ 'see', ' ', 'https://example.com/reports/q3?', 'lang=ar&mode=full', ' ', 'now', ]) }) test('keeps no-space ascii punctuation chains together as one breakable segment', () => { const prepared = prepareWithSegments('foo;bar foo:bar foo,bar as;lkdfjals;k', FONT) expect(prepared.segments).toEqual([ 'foo;bar', ' ', 'foo:bar', ' ', 'foo,bar', ' ', 'as;lkdfjals;k', ]) }) test('keeps numeric time ranges together', () => { const prepared = prepareWithSegments('window 7:00-9:00 only', FONT) expect(prepared.segments).toEqual(['window', ' ', '7:00-', '9:00', ' ', 'only']) }) test('splits hyphenated numeric identifiers at preferred boundaries', () => { const prepared = prepareWithSegments('SSN 420-69-8008 filed', FONT) expect(prepared.segments).toEqual(['SSN', ' ', '420-', '69-', '8008', ' ', 'filed']) }) test('keeps unicode-digit numeric expressions together', () => { const prepared = prepareWithSegments('यह २४×७ सपोर्ट है', FONT) expect(prepared.segments).toEqual(['यह', ' ', '२४×७', ' ', 'सपोर्ट', ' ', 'है']) }) test('does not attach opening punctuation to following whitespace', () => { const prepared = prepareWithSegments('“ hello', FONT) expect(prepared.segments).toEqual(['“', ' ', 'hello']) }) test('keeps japanese iteration marks attached to the preceding kana', () => { const prepared = prepareWithSegments('棄てゝ行く', FONT) expect(prepared.segments).toEqual(['棄', 'てゝ', '行', 'く']) }) test('carries trailing cjk opening punctuation forward across segment boundaries', () => { const prepared = prepareWithSegments('作者はさつき、「下人', FONT) expect(prepared.segments).toEqual(['作', '者', 'は', 'さ', 'つ', 'き、', '「下', '人']) }) test('keeps em dashes breakable', () => { const prepared = prepareWithSegments('universe—so', FONT) expect(prepared.segments).toEqual(['universe', '—', 'so']) }) test('coalesces repeated punctuation runs into a single segment', () => { const prepared = prepareWithSegments('=== heading ===', FONT) expect(prepared.segments).toEqual(['===', ' ', 'heading', ' ', '===']) }) test('applies CJK and Hangul punctuation attachment rules', () => { expect(prepareWithSegments('中文，测试。', FONT).segments).toEqual(['中', '文，', '测', '试。']) expect(prepareWithSegments('테스트입니다.', FONT).segments.at(-1)).toBe('다.') }) test('adjacent CJK text units stay breakable after visible text, not only after spaces', () => { const prepared = prepareWithSegments('foo 世界 bar', FONT) expect(prepared.segments).toEqual(['foo', ' ', '世', '界', ' ', 'bar']) const width = prepared.widths[0]! + prepared.widths[1]! + prepared.widths[2]! + 0.1 const batched = layoutWithLines(prepared, width, LINE_HEIGHT) expect(batched.lines.map(line => line.text)).toEqual(['foo 世', '界 bar']) const streamed = [] let cursor = { segmentIndex: 0, graphemeIndex: 0 } while (true) { const line = layoutNextLine(prepared, cursor, width) if (line === null) break streamed.push(line.text) cursor = line.end } expect(streamed).toEqual(['foo 世', '界 bar']) expect(layout(prepared, width, LINE_HEIGHT)).toEqual({ lineCount: 2, height: LINE_HEIGHT * 2 }) }) test('treats astral CJK ideographs as CJK break units', () => { expect(prepareWithSegments('𠀀𠀁', FONT).segments).toEqual(['𠀀', '𠀁']) expect(prepareWithSegments('𠀀。', FONT).segments).toEqual(['𠀀。']) }) test('prepare and prepareWithSegments agree on layout behavior', () => { const plain = prepare('Alpha beta gamma', FONT) const rich = prepareWithSegments('Alpha beta gamma', FONT) for (const width of [40, 80, 200]) { expect(layout(plain, width, LINE_HEIGHT)).toEqual(layout(rich, width, LINE_HEIGHT)) } }) test('locale can be reset without disturbing later prepares', () => { setLocale('th') const thai = prepare('ภาษาไทยภาษาไทย', FONT) expect(layout(thai, 80, LINE_HEIGHT).lineCount).toBeGreaterThan(0) setLocale(undefined) const latin = prepare('hello world', FONT) expect(layout(latin, 200, LINE_HEIGHT)).toEqual({ lineCount: 1, height: LINE_HEIGHT }) }) }) describe('layout invariants', () => { test('line count grows monotonically as width shrinks', () => { const prepared = prepare('The quick brown fox jumps over the lazy dog', FONT) let previous = 0 for (const width of [320, 200, 140, 90]) { const { lineCount } = layout(prepared, width, LINE_HEIGHT) expect(lineCount).toBeGreaterThanOrEqual(previous) previous = lineCount } }) test('trailing whitespace hangs past the line edge', () => { const prepared = prepareWithSegments('Hello ', FONT) const widthOfHello = prepared.widths[0]! expect(layout(prepared, widthOfHello, LINE_HEIGHT).lineCount).toBe(1) const withLines = layoutWithLines(prepared, widthOfHello, LINE_HEIGHT) expect(withLines.lineCount).toBe(1) expect(withLines.lines).toEqual([{ text: 'Hello', width: widthOfHello, start: { segmentIndex: 0, graphemeIndex: 0 }, a1:T2db2,Command: /bin/zsh -lc "sed -n '521,780p' src/layout.test.ts" Chunk ID: b677fb Wall time: 0.0000 seconds Process exited with code 0 Original token count: 2883 Output: end: { segmentIndex: 1, graphemeIndex: 0 }, }]) }) test('breaks long words at grapheme boundaries and keeps both layout APIs aligned', () => { const prepared = prepareWithSegments('Superlongword', FONT) const graphemeWidths = prepared.breakableWidths[0]! const maxWidth = graphemeWidths[0]! + graphemeWidths[1]! + graphemeWidths[2]! + 0.1 const plain = layout(prepared, maxWidth, LINE_HEIGHT) const rich = layoutWithLines(prepared, maxWidth, LINE_HEIGHT) expect(plain.lineCount).toBeGreaterThan(1) expect(rich.lineCount).toBe(plain.lineCount) expect(rich.height).toBe(plain.height) expect(rich.lines.map(line => line.text).join('')).toBe('Superlongword') expect(rich.lines[0]!.start).toEqual({ segmentIndex: 0, graphemeIndex: 0 }) expect(rich.lines.at(-1)!.end).toEqual({ segmentIndex: 1, graphemeIndex: 0 }) }) test('mixed-direction text is a stable smoke test', () => { const prepared = prepareWithSegments('According to محمد الأحمد, the results improved.', FONT) const result = layoutWithLines(prepared, 120, LINE_HEIGHT) expect(result.lineCount).toBeGreaterThanOrEqual(1) expect(result.height).toBe(result.lineCount * LINE_HEIGHT) expect(result.lines.map(line => line.text).join('')).toBe('According to محمد الأحمد, the results improved.') }) test('layoutNextLine reproduces layoutWithLines exactly', () => { const prepared = prepareWithSegments('foo trans\u00ADatlantic said "hello" to 世界 and waved.', FONT) const width = prepared.widths[0]! + prepared.widths[1]! + prepared.widths[2]! + prepared.breakableWidths[4]![0]! + prepared.discretionaryHyphenWidth + 0.1 const expected = layoutWithLines(prepared, width, LINE_HEIGHT) const actual = [] let cursor = { segmentIndex: 0, graphemeIndex: 0 } while (true) { const line = layoutNextLine(prepared, cursor, width) if (line === null) break actual.push(line) cursor = line.end } expect(actual).toEqual(expected.lines) }) test('rich line boundary cursors reconstruct normalized source text exactly', () => { const cases = [ 'a b c', ' Hello\t \n World ', 'foo trans\u00ADatlantic said "hello" to 世界 and waved.', 'According to محمد الأحمد, the results improved.', 'see https://example.com/reports/q3?lang=ar&mode=full now', 'alpha\u200Bbeta gamma', ] const widths = [40, 80, 120, 200] for (const text of cases) { const prepared = prepareWithSegments(text, FONT) const expected = prepared.segments.join('') for (const width of widths) { const batched = layoutWithLines(prepared, width, LINE_HEIGHT) const streamed = collectStreamedLines(prepared, width) expect(reconstructFromLineBoundaries(prepared, batched.lines)).toBe(expected) expect(reconstructFromLineBoundaries(prepared, streamed)).toBe(expected) expect(reconstructFromWalkedRanges(prepared, width)).toBe(expected) } } }) test('soft-hyphen round-trip uses source slices instead of rendered line text', () => { const prepared = prepareWithSegments('foo trans\u00ADatlantic', FONT) const width = prepared.widths[0]! + prepared.widths[1]! + prepared.widths[2]! + prepared.breakableWidths[4]![0]! + prepared.discretionaryHyphenWidth + 0.1 const result = layoutWithLines(prepared, width, LINE_HEIGHT) expect(result.lines.map(line => line.text).join('')).toBe('foo trans-atlantic') expect(reconstructFromLineBoundaries(prepared, result.lines)).toBe('foo trans\u00ADatlantic') }) test('pre-wrap mode keeps hanging spaces visible at line end', () => { const prepared = prepareWithSegments('foo bar', FONT, { whiteSpace: 'pre-wrap' }) const width = measureWidth('foo', FONT) + 0.1 const lines = layoutWithLines(prepared, width, LINE_HEIGHT) expect(lines.lineCount).toBe(2) expect(lines.lines.map(line => line.text)).toEqual(['foo ', 'bar']) expect(layout(prepared, width, LINE_HEIGHT).lineCount).toBe(2) }) test('pre-wrap mode treats hard breaks as forced line boundaries', () => { const prepared = prepareWithSegments('a\nb', FONT, { whiteSpace: 'pre-wrap' }) const lines = layoutWithLines(prepared, 200, LINE_HEIGHT) expect(lines.lines.map(line => line.text)).toEqual(['a', 'b']) expect(layout(prepared, 200, LINE_HEIGHT).lineCount).toBe(2) }) test('pre-wrap mode treats tabs as hanging whitespace aligned to tab stops', () => { const prepared = prepareWithSegments('a\tb', FONT, { whiteSpace: 'pre-wrap' }) const spaceWidth = measureWidth(' ', FONT) const prefixWidth = measureWidth('a', FONT) const tabAdvance = nextTabAdvance(prefixWidth, spaceWidth, 8) const textWidth = prefixWidth + tabAdvance + measureWidth('b', FONT) const width = textWidth - 0.1 const lines = layoutWithLines(prepared, width, LINE_HEIGHT) expect(lines.lines.map(line => line.text)).toEqual(['a\t', 'b']) expect(layout(prepared, width, LINE_HEIGHT).lineCount).toBe(2) }) test('pre-wrap mode treats consecutive tabs as distinct tab stops', () => { const prepared = prepareWithSegments('a\t\tb', FONT, { whiteSpace: 'pre-wrap' }) const spaceWidth = measureWidth(' ', FONT) const prefixWidth = measureWidth('a', FONT) const firstTabAdvance = nextTabAdvance(prefixWidth, spaceWidth, 8) const afterFirstTab = prefixWidth + firstTabAdvance const secondTabAdvance = nextTabAdvance(afterFirstTab, spaceWidth, 8) const width = prefixWidth + firstTabAdvance + secondTabAdvance - 0.1 const lines = layoutWithLines(prepared, width, LINE_HEIGHT) expect(lines.lines.map(line => line.text)).toEqual(['a\t\t', 'b']) expect(layout(prepared, width, LINE_HEIGHT).lineCount).toBe(2) }) test('pre-wrap mode keeps whitespace-only middle lines visible', () => { const prepared = prepareWithSegments('foo\n \nbar', FONT, { whiteSpace: 'pre-wrap' }) const lines = layoutWithLines(prepared, 200, LINE_HEIGHT) expect(lines.lines.map(line => line.text)).toEqual(['foo', ' ', 'bar']) expect(layout(prepared, 200, LINE_HEIGHT)).toEqual({ lineCount: 3, height: LINE_HEIGHT * 3 }) }) test('pre-wrap mode keeps trailing spaces before a hard break on the current line', () => { const prepared = prepareWithSegments('foo \nbar', FONT, { whiteSpace: 'pre-wrap' }) const lines = layoutWithLines(prepared, 200, LINE_HEIGHT) expect(lines.lines.map(line => line.text)).toEqual(['foo ', 'bar']) expect(layout(prepared, 200, LINE_HEIGHT)).toEqual({ lineCount: 2, height: LINE_HEIGHT * 2 }) }) test('pre-wrap mode keeps trailing tabs before a hard break on the current line', () => { const prepared = prepareWithSegments('foo\t\nbar', FONT, { whiteSpace: 'pre-wrap' }) const lines = layoutWithLines(prepared, 200, LINE_HEIGHT) expect(lines.lines.map(line => line.text)).toEqual(['foo\t', 'bar']) expect(layout(prepared, 200, LINE_HEIGHT)).toEqual({ lineCount: 2, height: LINE_HEIGHT * 2 }) }) test('pre-wrap mode restarts tab stops after a hard break', () => { const prepared = prepareWithSegments('foo\n\tbar', FONT, { whiteSpace: 'pre-wrap' }) const lines = layoutWithLines(prepared, 200, LINE_HEIGHT) const spaceWidth = measureWidth(' ', FONT) const expectedSecondLineWidth = nextTabAdvance(0, spaceWidth, 8) + measureWidth('bar', FONT) expect(lines.lines.map(line => line.text)).toEqual(['foo', '\tbar']) expect(lines.lines[1]!.width).toBeCloseTo(expectedSecondLineWidth, 5) }) test('layoutNextLine stays aligned with layoutWithLines in pre-wrap mode', () => { const prepared = prepareWithSegments('foo\n bar baz\nquux', FONT, { whiteSpace: 'pre-wrap' }) const width = measureWidth(' bar', FONT) + 0.1 const expected = layoutWithLines(prepared, width, LINE_HEIGHT) const actual = [] let cursor = { segmentIndex: 0, graphemeIndex: 0 } while (true) { const line = layoutNextLine(prepared, cursor, width) if (line === null) break actual.push(line) cursor = line.end } expect(actual).toEqual(expected.lines) }) test('pre-wrap mode keeps empty lines from consecutive hard breaks', () => { const prepared = prepareWithSegments('\n\n', FONT, { whiteSpace: 'pre-wrap' }) const lines = layoutWithLines(prepared, 200, LINE_HEIGHT) expect(lines.lines.map(line => line.text)).toEqual(['', '']) expect(layout(prepared, 200, LINE_HEIGHT)).toEqual({ lineCount: 2, height: LINE_HEIGHT * 2 }) }) test('pre-wrap mode does not invent an extra trailing empty line', () => { const prepared = prepareWithSegments('a\n', FONT, { whiteSpace: 'pre-wrap' }) const lines = layoutWithLines(prepared, 200, LINE_HEIGHT) expect(lines.lines.map(line => line.text)).toEqual(['a']) expect(layout(prepared, 200, LINE_HEIGHT)).toEqual({ lineCount: 1, height: LINE_HEIGHT }) }) test('overlong breakable segments wrap onto a fresh line when the current line already has content', () => { const prepared = prepareWithSegments('foo abcdefghijk', FONT) const prefixWidth = prepared.widths[0]! + prepared.widths[1]! const wordBreaks = prepared.breakableWidths[2]! const width = prefixWidth + wordBreaks[0]! + wordBreaks[1]! + 0.1 const batched = layoutWithLines(prepared, width, LINE_HEIGHT) expect(batched.lines[0]?.text).toBe('foo ') expect(batched.lines[1]?.text.startsWith('ab')).toBe(true) const streamed = layoutNextLine(prepared, { segmentIndex: 0, graphemeIndex: 0 }, width) expect(streamed?.text).toBe('foo ') expect(layout(prepared, width, LINE_HEIGHT).lineCount).toBe(batched.lineCount) }) test('walkLineRanges reproduces layoutWithLines geometry without materializing text', () => { const prepared = prepareWithSegments('foo trans\u00ADatlantic said "hello" to 世界 and waved.', FONT) const width = prepared.widths[0]! + prepared.widths[1]! + prepared.widths[2]! + prepared.breakableWidths[4]![0]! + prepared.discretionaryHyphenWidth + 0.1 const expected = layoutWithLines(prepared, width, LINE_HEIGHT) const actual: Array<{ width: number start: { segmentIndex: number, graphemeIndex: number } end: { segmentIndex: number, graphemeIndex: number } }> = [] const lineCount = walkLineRanges(prepared, width, line => { actual.push({ width: line.width, start: { ...line.start }, end: { ...line.end }, }) }) expect(lineCount).toBe(expected.lineCount) expect(actual).toEqual(expected.lines.map(line => ({ width: line.width, start: line.start, end: line.end, }))) }) test('countPreparedLines stays aligned with the walked line counter', () => { const texts = [ 'The quick brown fox jumps over the lazy dog.', 'said "hello" to 世界 and waved.', 'مرحبا، عالم؟', 'author 7:00-9:00 only', 'alpha\u200Bbeta gamma', ] const widths = [40, 80, 120, 200] for (let textIndex = 0; textIndex < texts.length; textIndex++) { const prepared = prepareWithSegments(texts[textIndex]!, FONT) for (let widthIndex = 0; widthIndex < widths.length; widthIndex++) { const width = widths[widthIndex]! const counted = countPreparedLines(prepared, width) const walked = walkPreparedLines(prepared, width) expect(counted).toBe(walked) } } }) }) a2:Tbde,Command: /bin/zsh -lc "sed -n '1,220p' STATUS.md" Chunk ID: 7c09ea Wall time: 0.0000 seconds Process exited with code 0 Original token count: 721 Output: # Current Status Machine-readable current snapshot data for the main browser sweep and benchmark numbers lives in JSON. Use this file for "where do I look right now?". Use [RESEARCH.md](/Users/chenglou/github/pretext/RESEARCH.md) for why the numbers changed and what was tried. Use [corpora/STATUS.md](/Users/chenglou/github/pretext/corpora/STATUS.md) for the long-form corpus canaries. ## Browser Accuracy Machine-readable dashboard: - [status/dashboard.json](/Users/chenglou/github/pretext/status/dashboard.json) Raw snapshots: - [accuracy/chrome.json](/Users/chenglou/github/pretext/accuracy/chrome.json) - [accuracy/safari.json](/Users/chenglou/github/pretext/accuracy/safari.json) - [accuracy/firefox.json](/Users/chenglou/github/pretext/accuracy/firefox.json) Notes: - This is the 4-font × 8-size × 8-width × 30-text browser corpus. - The public accuracy page is effectively a regression gate now, not the main steering metric. ## Benchmark Snapshot Machine-readable dashboard: - [status/dashboard.json](/Users/chenglou/github/pretext/status/dashboard.json) Raw snapshots: - [benchmarks/chrome.json](/Users/chenglou/github/pretext/benchmarks/chrome.json) - [benchmarks/safari.json](/Users/chenglou/github/pretext/benchmarks/safari.json) Notes: - Chrome remains the main maintained performance baseline. Safari snapshots are still useful, but they are noisier and warm up less predictably. - The checked-in JSON snapshots are cold checker runs. Ad hoc page-driven numbers, especially in Safari, can differ after warmup. - Refresh the benchmark JSON snapshots when a diff changes benchmark methodology or the text engine's hot path (`src/analysis.ts`, `src/measurement.ts`, `src/line-break.ts`, `src/layout.ts`, `src/bidi.ts`, or `pages/benchmark.ts`). - `layout()` remains the resize hot path; `prepare()` is where script-specific cost still lives. - Long-form corpus benchmark rows split `prepare()` into analysis and measurement phases, which helps separate segmentation/glue cost from raw width-measurement cost. ## Pointers - Main dashboard summary: [status/dashboard.json](/Users/chenglou/github/pretext/status/dashboard.json) - Accuracy snapshots: [accuracy/chrome.json](/Users/chenglou/github/pretext/accuracy/chrome.json), [accuracy/safari.json](/Users/chenglou/github/pretext/accuracy/safari.json), [accuracy/firefox.json](/Users/chenglou/github/pretext/accuracy/firefox.json) - Benchmark snapshots: [benchmarks/chrome.json](/Users/chenglou/github/pretext/benchmarks/chrome.json), [benchmarks/safari.json](/Users/chenglou/github/pretext/benchmarks/safari.json) - Corpus dashboard: [corpora/dashboard.json](/Users/chenglou/github/pretext/corpora/dashboard.json) - Corpus anchor rows: [corpora/representative.json](/Users/chenglou/github/pretext/corpora/representative.json) - Full exploration log: [RESEARCH.md](/Users/chenglou/github/pretext/RESEARCH.md) a3:Tb74,Command: /bin/zsh -lc "sed -n '1,220p' TODO.md" Chunk ID: 0a53b0 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 695 Output: Current priorities: 1. Keep the canaries honest - Mixed app text still has the extractor-sensitive `710px` soft-hyphen miss. - Chinese is still the clearest active CJK canary: Safari anchors are clean, while Chrome keeps a broader narrow-width positive field with real font sensitivity. - Myanmar and Urdu remain useful shaping/context canaries, but they are not the active tuning target right now. 2. Next engine work - Use the split `analyze()` / `measure()` benchmark rows to steer any remaining `prepare()` work instead of reopening generic profiling. - Expand mixed app text only when it adds a real product-shaped class, e.g. URL/query runs, mixed bidi with numbers, emoji ZWJ runs, or `NBSP` / `ZWSP` / `WJ` behavior. - Broaden canaries only when the source text is clean. - If we add another Southeast Asian canary, prefer a clean source text that broadens the class instead of another wrapped/legal/raw-source artifact. - Expand the sampled font matrix only where a canary still looks genuinely imperfect. - Treat strongly font-sensitive or shaping-sensitive misses as boundary-finding for the current architecture, not automatic invitations for another local glue rule. - Keep the hot `layout()` path simple and allocation-light while the rich path absorbs more userland layout needs. 3. Demo work - Keep the editorial demos as the dogfood path for the rich line APIs. - Prefer `layoutNextLine()` / `walkLineRanges()` when a demo is really about streaming or obstacle-aware layout. - Add a new demo only if it exposes something the current editorial demos do not already cover. Not worth doing right now: - Do not chase universal exactness as the product claim. - Do not put measurement back in `layout()`. - Do not resurrect dirty corpora just to cover another language. - Do not overfit one-line misses in one browser/corpus without broader evidence. - Do not let browser-profile shims turn into a grab bag of ad hoc engine knobs. - Do not explode the public API with cache or engine knobs. Still-open design questions: - Whether line-fit tolerance should stay as a browser shim or move toward runtime calibration. - Whether `{ whiteSpace: 'pre-wrap' }` should grow beyond spaces / tabs / `\n`. - Whether strong real-world demand for `system-ui` would justify a narrow prepare-time DOM fallback. - Whether server canvas support should become an explicit supported backend. - Whether automatic hyphenation beyond manual soft hyphen is in scope. - Whether intrinsic sizing / logical width APIs are needed beyond fixed-width height prediction. - Whether bidi rendering concerns like selection and copy/paste belong here or stay out of scope. - Whether a separate optional slow verify path is worth having as a diagnostic mode, without contaminating `layout()`. a4:T4379,Command: /bin/zsh -lc "git log --reverse --date=short --pretty=format:'%h %ad %s'" Chunk ID: 0e1cbf Wall time: 0.0001 seconds Process exited with code 0 Original token count: 4272 Output: f962f72 2026-03-03 Initial project setup f0f03ba 2026-03-03 Add oxlint config with type-aware linting 1d3941e 2026-03-03 Add accuracy sweep page dfd40ed 2026-03-03 Add headless test suite with HarfBuzz measurement d399fc4 2026-03-03 Add word-sum vs full-line accuracy test 0167c1e 2026-03-03 Fix HarfBuzz direction: explicit LTR for consistent measurement 820e8e1 2026-03-03 Add comments documenting motivations throughout c78b35a 2026-03-03 Write README documenting problem, solution, perf, accuracy, i18n, limitations, how it works, what we tried and rejected, and credits ead6bd7 2026-03-03 Update CLAUDE.md with test commands, file inventory, rejected approaches a797983 2026-03-03 Remove serve.ts, use bun's direct HTML serving 61c1e25 2026-03-03 Add RESEARCH.md, consolidate docs, rename serve to start 73783d0 2026-03-03 Add line-break diagnostic, document Safari CSS behavior differences f54a344 2026-03-03 Revert trailing space experiment — restore Chrome 99.4% 08a1404 2026-03-03 Document Safari CSS line-breaking differences in RESEARCH.md e0837f6 2026-03-03 Update CLAUDE.md and README for fresh session handoff 91304c0 2026-03-03 Emoji correction, merging fix, overflow handling — Chrome 99.4% → 99.7% 6ce9982 2026-03-03 Kinsoku shori — CJK line-breaking rules, Chrome 99.7% → 99.9% 315658b 2026-03-03 Document cache design decision, slim down CLAUDE.md e8de24c 2026-03-03 Fix prepare() perf regression: skip emoji grapheme scan for non-emoji text ae466fa 2026-03-03 Hoist word segmenter to module level, add TODO section aff5ba7 2026-03-03 Benchmark CJK scaling, cleanup for...of, --no-hmr, NaN guard fee6750 2026-03-03 Opening bracket forward-merge, CSS config docs, Chrome 3838/3840 22a2f26 2026-03-03 Document word-sum divergence, revert failed fix attempts e2f1321 2026-03-03 Document char-level experiment, prior art, fix text-layout repo URL c48295a 2026-03-03 Document hybrid verify experiment (99.9% → 99.8%, reverted) 573b6db 2026-03-04 Fix emoji correction for Safari — compare canvas vs DOM, not fontSize 3fd4f01 2026-03-04 Document lineHeight default mismatch across browsers 71c0e58 2026-03-04 Document Firefox system-ui font mismatch and Thai segmenter divergence 1135318 2026-03-04 Add browser bug links to RESEARCH.md, update emoji correction docs bad8fbc 2026-03-04 CJK-split all CJK segments, not just word-like — Firefox 99.7% → 99.9% 48116bd 2026-03-04 Add Firefox system-ui bug link to RESEARCH.md 038a812 2026-03-04 Add Chrome system-ui bug link to RESEARCH.md 5f76956 2026-03-04 Add Verdana and Courier New to accuracy sweep — 7677/7680 (99.96%) b22efa0 2026-03-04 Show 2 decimal places for accuracy percentage 76c2766 2026-03-04 Update all docs for 4-font accuracy, clean up stale comments 7d0a54d 2026-03-04 Remove checked in chromium issue 5ecce72 2026-03-04 Simplify layout code: flatten data, remove dead computation, cut allocations 965a546 2026-03-04 Update RESEARCH.md with Firefox developer's DPR explanation 06cb7d0 2026-03-04 Add bubble shrinkwrap demo page f99df17 2026-03-06 Remove interleaving demo stub dbebb7d 2026-03-06 Clean up stale docs and unify test data fea1fa7 2026-03-06 Split fast and rich layout APIs 0aca7b6 2026-03-06 Link system-ui warnings to RESEARCH.md exploration 7a4f827 2026-03-06 Make lineHeight an explicit layout-time input 9d37c3a 2026-03-07 Trim CLAUDE.md to internal notes 89ed442 2026-03-07 Document public layout data fields 71d5ff4 2026-03-07 Improve benchmark methodology for tiny layout timings 3eabf39 2026-03-07 TODOs and more use-cases 374aece 2026-03-07 Rename project from text-metrics to pretext bf0741a 2026-03-07 Add unported ancestor ideas to notes b1c4041 2026-03-07 Move harfbuzzjs from dependencies to devDependencies 81f7bf1 2026-03-07 Clarify break-word docs 31af255 2026-03-07 Clarify CSS config claim and server-side status, add TODOs a724b7f 2026-03-07 Update benchmark snapshot d07dd7a 2026-03-07 gatsby test: first pass 5cd5762 2026-03-07 Add Gatsby diagnostics without regressing hot-path layout 9af7f63 2026-03-07 Close the remaining browser accuracy gaps cheaply 8f4ac00 2026-03-07 Harden browser-accuracy diagnostics without slowing layout 3f9cb1e 2026-03-08 Separate prepare analysis from measurement 84cccc8 2026-03-08 Shrink permanent tests to exported invariants 7137f0a 2026-03-08 Add a fast Gatsby width sweep dfd81ae 2026-03-08 Harden Gatsby sweep browser automation 06e6437 2026-03-08 Keep apostrophe-led elisions with the following word c852293 2026-03-08 Simplify merged segmentation and CJK measurement 3202f3a 2026-03-08 Add multilingual corpus stress canaries 16c04fe 2026-03-08 Improve Arabic and Hindi corpus wrapping 516e5e6 2026-03-08 Make bun start clear stale port 3000 listeners bba3624 2026-03-08 Benchmark long-form corpora alongside the shared corpus cb0a76a 2026-03-08 Harden corpus mismatch diagnostics 7fb726d 2026-03-08 Fingerprint browser test environments and fix Korean quote wraps in Chromium cdec07c 2026-03-08 Clean Arabic corpus artifacts and harden RTL diagnostics bc031c3 2026-03-08 Document the rejected Arabic shaping experiments e7b9cbe 2026-03-08 Expose logical break offsets in corpus diagnostics 52fcf70 2026-03-09 Add a single-snippet browser line-break probe cd8b642 2026-03-09 Rename CLAUDE.md to AGENTS.md d91a0af 2026-03-09 Merge Arabic no-space punctuation clusters 9612485 2026-03-09 Refine Arabic preprocessing around punctuation and marks 1495b5c 2026-03-09 Fix multilingual corpus diagnostics and clean Arabic source artifacts 09e6c03 2026-03-09 Clean remaining Arabic corpus punctuation artifacts 8aaf7b8 2026-03-09 Keep Arabic punctuation-plus-mark clusters with their word 49e0f36 2026-03-09 Loosen non-Safari edge tolerance for remaining Arabic fits dcd410b 2026-03-09 Add Arabic and Hebrew stress corpora 30b1220 2026-03-09 Trim editorial noise from Hebrew stress corpus 9b59710 2026-03-09 Add sampled sweep mode for large corpora 4715123 2026-03-09 Add Thai stress corpus and fix ASCII quote wrapping 3e991df 2026-03-09 Share line breaking between layout APIs d988bb0 2026-03-09 Keep layout on a specialized fast path 62b258a 2026-03-10 Strengthen break semantics beyond plain spaces c340d09 2026-03-10 Cache richer segment metrics during prepare d9f9873 2026-03-10 Model discretionary soft hyphen breaks 7995c72 2026-03-10 Add a mixed app-text canary and glue URL runs 86f741f 2026-03-10 Refine mixed app-text breaking and diagnostics b7ec1a8 2026-03-10 Add a Khmer stress corpus and benchmark coverage 5e92358 2026-03-10 Make prepare() return an opaque handle 396caff 2026-03-10 Add corpus font-matrix checks and font overrides ccec343 2026-03-10 Reject the Lao raw-law canary and add a clean Myanmar corpus c9cbfca 2026-03-10 Tighten Myanmar punctuation glue and probe diagnostics b6e2104 2026-03-10 Finish the Myanmar phrase-glue canary b89a7fb 2026-03-10 Make layout imports safe outside the DOM 09a5f8c 2026-03-10 Give shared Safari automation its own window c94b2ba 2026-03-10 Improve SHY diagnostics and line rendering 78a64be 2026-03-10 Add a second clean Thai stress corpus 115558d 2026-03-10 Document rejected Myanmar break experiments 1b32ece 2026-03-10 Add normalized slice mode to corpus diagnostics 7c37dfb 2026-03-10 Add a second Myanmar stress corpus a87e756 2026-03-10 Add a compact corpus status snapshot 35d4ff8 2026-03-10 Separate current snapshots from historical notes d8e7c64 2026-03-10 Refresh current snapshots and benchmark corpus rows 93b10a6 2026-03-10 Keep bidi metadata off the fast prepare path 830d8ad 2026-03-10 Add a Japanese prose canary and fix iteration marks 2c17aec 2026-03-10 Stop browser checkers from polling the active tab 0c30ea1 2026-03-10 Keep benchmark runs foregrounded e30c04e 2026-03-10 Extract bidi logic from layout da60460 2026-03-10 Split benchmark prepare timings by phase c4cf982 2026-03-10 Split prepare internals and add a second Japanese canary 92b1192 2026-03-10 Turn the demo page into a manual line layout lab f11f7e8 2026-03-10 Add an Urdu Nastaliq canary a734f17 2026-03-10 Expose discretionary hyphen state to userland renderers d782b14 2026-03-10 Add explicit locale control for text analysis bc6174f 2026-03-10 Add a three-column userland reflow demo 614dfab 2026-03-10 Add a line-by-line reflow API and contour demo 4a6a457 2026-03-10 Add a Chinese long-form canary a5f94ba 2026-03-10 Add a second Chinese prose canary 20134b9 2026-03-10 Serialize browser automation per engine f06fb75 2026-03-10 Extract the internal line-break core 57964f9 2026-03-11 Add corpus taxonomy tooling 90f1624 2026-03-11 Add an editorial reflow demo and carry CJK opening punctuation bd0be6f 2026-03-11 Expose the corpus taxonomy runner as a bun script c9796c5 2026-03-11 Add a synced multi-view reflow demo 81526d3 2026-03-11 Highlight shared anchors across synced reflow panes 4e4f71e 2026-03-11 Dogfood layoutNextLine in the browser demos 3fa02c2 2026-03-11 Center the shared anchor in the sync demo da88d5c 2026-03-11 Stream columns directly from layoutNextLine cac7a1b 2026-03-11 Record rejected Chinese punctuation experiments 4526cde 2026-03-11 Add a winding-contour editorial demo 2a11156 2026-03-11 Rebuild Chronicle around an explicit render pass f43a585 2026-03-11 Add a viewport-warped polygon blob lab f472d70 2026-03-11 Simplify blob lab to tapered black silhouettes a9652fd 2026-03-11 Add a batch low-level line geometry API 2be6a3a 2026-03-11 Add draft project notes b2dc7c8 2026-03-11 Add a two-logo contour layout demo 84c0c96 2026-03-11 Tighten logo column contour spacing f6fe933 2026-03-11 Refine the logo columns layout a0bc8be 2026-03-11 Adapt the logo demo copy from Situational Awareness 4d4f561 2026-03-11 test 12daa0e 2026-03-11 Use full reload for the dev page server a98fc00 2026-03-12 Make logo columns interaction geometry-driven fd9e9d1 2026-03-12 Project logo columns through a DOM cache 28a24bd 2026-03-12 Tune logo columns for narrower spreads b0f80c2 2026-03-12 Extract generic wrap geometry from logo columns 597500b 2026-03-12 Polish the logo columns editorial spread cc6ee94 2026-03-12 Wait for logo hulls before first render 4bdf313 2026-03-12 Route the logo spread title around the logo geometry 87d509c 2026-03-12 Consolidate the editorial demos around dynamic layout 938e96f 2026-03-12 Document break-word behavior and trim dynamic layout overhead c72a1c7 2026-03-12 Keep dynamic layout work in render 45679aa 2026-03-12 Make emoji correction lazy for plain text b6ad165 2026-03-12 Quantize dynamic layout headline sizing 9de553c 2026-03-12 Make dynamic layout bootstrap explicit be96850 2026-03-12 Flatten dynamic layout control loops 035bf7c 2026-03-12 Simplify polygon scanline intersections 20c270a 2026-03-12 Remove stale demos and restore text cursor cdb56ab 2026-03-12 More public docs cae482d 2026-03-12 Clarify rich bidi metadata and share diagnostics 4fdb790 2026-03-12 Refresh status docs and tighten stale comments ee8ddac 2026-03-12 Clean up readmes b4bdd1c 2026-03-17 Rewritten readme for public consumptions 335c2c7 2026-03-17 check in TODOs 656c910 2026-03-17 Fix stale README glossary and improve use-case 2 examples bbc5fca 2026-03-17 Update bubbles demo 3e8352e 2026-03-20 Broaden internal CJK coverage 54d0206 2026-03-20 Document text-shaper comparison 40bd19f 2026-03-22 Move demos into pages/demos/, remove demo page 0a84e4b 2026-03-22 Add masonry demo: virtualized text layout with pretext 76f3864 2026-03-22 Masonry: 10k cards with array-based domCache 8b8a027 2026-03-23 Masonry: fix centering, proportional column widths, 10k cards ffc2a75 2026-03-23 Masonry: add DOCTYPE, remove debug, fix virtualization d346ba8 2026-03-23 Add LAN demo server script 837f080 2026-03-23 Add iOS Safari scroll crash repro demos 2fbae37 2026-03-23 Remove iOS Safari scroll crash repro demos a0f7c66 2026-03-23 Clean up masonry demo control flow 06de9d7 2026-03-26 Simplify masonry DOM ownership 2823649 2026-03-26 Refresh benchmark snapshots and simplify rich line API b11f2b2 2026-03-26 Add raw accuracy and corpus snapshots bc8e0db 2026-03-26 Refine rich layout internals and docs 0662f0b 2026-03-26 Prepare npm package surface d2ea9b2 2026-03-26 Fix Safari punctuation-chain wrapping 6b7b0e4 2026-03-26 Let overlong breakable segments fill the current line 7edfa07 2026-03-26 Clarify Safari extractor-sensitive diagnostics 3195cb3 2026-03-26 Tune Safari line breaking and add input-box demo 2aeecec 2026-03-27 Refresh snapshots and add changelog 4d3d184 2026-03-27 Document system-ui measurement conclusion f787f34 2026-03-27 Replace input demo with accordion demo 2da7da3 2026-03-27 Polish bubbles demo and stabilize dev server b7323f2 2026-03-27 Release 0.0.1 a7f6bab 2026-03-27 Tighten changelog wording 22d2fac 2026-03-27 Publish demos to GitHub Pages e0a35de 2026-03-27 Fix Pages build on GitHub Actions 489e4b1 2026-03-27 Enable GitHub Pages from workflow fc78d69 2026-03-27 Tighten bubbles demo copy 4bb6491 2026-03-27 Refresh bubbles demo conversation 7097a8e 2026-03-27 Add variable typographic ASCII demo 5d631d1 2026-03-27 Add wireframe torus demo 415f0a8 2026-03-27 Remove wireframe torus demo 0748356 2026-03-27 Add calligram engine demo f144a8c 2026-03-27 Remove calligram engine demo b01f83d 2026-03-27 Add editorial engine demo 2f2ef13 2026-03-27 Update demos index links and README 8257986 2026-03-27 Simplify bubbles startup and runtime 24ad012 2026-03-27 Fix dynamic layout assets and root demo links cd04257 2026-03-27 Fix + add max' links 6b8d03a 2026-03-27 Update bubbles bidi example copy cfdc371 2026-03-27 Add preserve-spaces whitespace mode a28df8c 2026-03-27 Rewrite whitespace mode around pre-wrap chunks 784ee53 2026-03-27 Add pre-wrap oracle validation 98a1ad5 2026-03-27 Add tab stops to pre-wrap mode 8bd147d 2026-03-27 Broaden pre-wrap oracle coverage 0c3c8ce 2026-03-28 Add somnai attribution to demo pages 2fa99fa 2026-03-28 Simplify pre-wrap tabs and broaden coverage bdb0f95 2026-03-28 Broaden pre-wrap newline coverage a1ed28e 2026-03-28 Refresh benchmark snapshots for pre-wrap branch d18bb75 2026-03-28 Restore simple layout fast path 38c7090 2026-03-28 Split simple rich line walking from pre-wrap path 44dc489 2026-03-28 Add checked-in rich path benchmarks 8e305a7 2026-03-28 Only compute prefix widths when the browser needs them 4c634ca 2026-03-28 Merge branch 'preserve-whitespace' a06fc7f 2026-03-28 Allow selecting editorial demo text 9793990 2026-03-28 Release 0.0.2 2604c28 2026-03-28 Trim internal docs 9754408 2026-03-28 Remove bad link ddce81e 2026-03-28 Bump dev deps and allow TypeScript 6 212113e 2026-03-28 Restore masonry text selection e8b7ace 2026-03-28 Add rich text demo 4eeed18 2026-03-28 Improve dynamic layout mobile mode 9f75eba 2026-03-28 Improve editorial demo mobile mode d79f0f7 2026-03-28 Remove *dev time* harfbuzz scripts so that X dummies don't think this is a harfbuzz wrapper 0363564 2026-03-28 Disable dynamic layout mobile rubber-banding 51b54f9 2026-03-28 Allow dynamic layout text selection on mobile 15f9db5 2026-03-28 Allow editorial text selection on mobile 1dda66b 2026-03-28 Refine editorial selection interaction state 75a6e48 2026-03-29 Publish built ESM package artifacts 4ec8481 2026-03-29 Merge PR #4 (superseded by final packaging implementation) d618e84 2026-03-29 Consolidate development command docs 19451bc 2026-03-29 Release 0.0.3 53a065b 2026-03-29 Trim duplicated docs guidance a8b8bfb 2026-03-29 Refresh TODO priorities a61a101 2026-03-29 Expand TODO notes 92c51d4 2026-03-29 fix(demo): text deselection in demos by skipping unchanged textContent writes 0d6660e 2026-03-29 Preserve demo text DOM during selection 77dabd5 2026-03-29 Use spans for manual text lines 45e447b 2026-03-29 Inline prepare() vs layout() tip in README 04f7ac0 2026-03-29 Unify Firefox accuracy automation 049244d 2026-03-29 Batch corpus sweeps and simplify report transport 3ca1e6d 2026-03-29 Add posted reports for batched corpus sweeps 1fe1201 2026-03-29 Batch Gatsby sweeps in page 7aaff29 2026-03-29 Document batched sweep automation 25121a3 2026-03-29 Support even sampling in gatsby sweeps ae99378 2026-03-29 Add phase-aware automation timeout diagnostics 588293c 2026-03-29 Batch corpus taxonomy automation 89cd7f6 2026-03-29 Document corpus taxonomy batching b042fac 2026-03-30 refactor: split justification-comparison into .html + .ts 2558f05 2026-03-29 Merge pull request #26 from somnai-dreams/demo/justification-comparison-split 16e228d 2026-03-29 Add justification comparison demo to demos 12688f0 2026-03-29 Polish justification comparison demo d5eac0b 2026-03-30 Refactor justification comparison demo 91d2f78 2026-03-30 Generate corpus status from sweep snapshots d178a84 2026-03-30 Replace markdown status tables with JSON dashboards 39b38e5 2026-03-31 Add rich line round-trip invariants 325c0af 2026-03-31 Trim Gatsby round-trip diagnostics b232ac2 2026-03-31 Sanitize masonry demo text artifacts 0f889e6 2026-03-31 Route Gatsby through corpus tooling and fix stale pending breaks aaed2f7 2026-03-31 Normalize masonry demo dataset offline a8d1e35 2026-03-31 Keep correctness browser automation in backgrounda5:T5c6,Command: /bin/zsh -lc "sed -n '1,260p' CHANGELOG.md" Chunk ID: 92c790 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 331 Output: # Changelog ## Unreleased ## 0.0.3 - 2026-03-29 ### Changed - npm now publishes built ESM JavaScript from `dist/` instead of exposing raw TypeScript source as the package entrypoint. - TypeScript consumers now pick up shipped declaration files automatically from the published package, while plain JavaScript consumers can install and import the package without relying on dependency-side TypeScript transpilation. ## 0.0.2 - 2026-03-28 ### Added - `{ whiteSpace: 'pre-wrap' }` mode for textarea-like text, preserving ordinary spaces, tabs, and hard breaks. ## 0.0.1 - 2026-03-27 ### Changed - Safari line breaking now has a clearer browser-specific policy path for narrow soft-hyphen and breakable-run cases. - Browser tooling is more stable: fresh per-run page ports, diagnostics derived from the public rich layout API, and a non-watch `bun start` by default. ## 0.0.0 - 2026-03-26 Initial public npm release of `@chenglou/pretext`. ### Added - `prepare()` and `layout()` as the core fast path for DOM-free multiline text height prediction. - Rich layout APIs including `prepareWithSegments()`, `layoutWithLines()`, `layoutNextLine()`, and `walkLineRanges()` for custom rendering and manual layout. - Browser accuracy, benchmark, and corpus tooling with checked-in snapshots and representative canaries. a6:T15c9,Command: /bin/zsh -lc "git log --date=short --pretty=format:'%ad %h %s' -- src/layout.ts src/analysis.ts src/line-break.ts src/measurement.ts src/bidi.ts | head -n 120" Chunk ID: 2df792 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1326 Output: 2026-03-31 0f889e6 Route Gatsby through corpus tooling and fix stale pending breaks 2026-03-31 39b38e5 Add rich line round-trip invariants 2026-03-29 75a6e48 Publish built ESM package artifacts 2026-03-28 8e305a7 Only compute prefix widths when the browser needs them 2026-03-28 38c7090 Split simple rich line walking from pre-wrap path 2026-03-28 d18bb75 Restore simple layout fast path 2026-03-28 2fa99fa Simplify pre-wrap tabs and broaden coverage 2026-03-27 98a1ad5 Add tab stops to pre-wrap mode 2026-03-27 a28df8c Rewrite whitespace mode around pre-wrap chunks 2026-03-27 cfdc371 Add preserve-spaces whitespace mode 2026-03-26 3195cb3 Tune Safari line breaking and add input-box demo 2026-03-26 6b7b0e4 Let overlong breakable segments fill the current line 2026-03-26 d2ea9b2 Fix Safari punctuation-chain wrapping 2026-03-26 bc8e0db Refine rich layout internals and docs 2026-03-26 2823649 Refresh benchmark snapshots and simplify rich line API 2026-03-20 3e8352e Broaden internal CJK coverage 2026-03-12 4fdb790 Refresh status docs and tighten stale comments 2026-03-12 cae482d Clarify rich bidi metadata and share diagnostics 2026-03-12 45679aa Make emoji correction lazy for plain text 2026-03-11 a9652fd Add a batch low-level line geometry API 2026-03-11 90f1624 Add an editorial reflow demo and carry CJK opening punctuation 2026-03-11 57964f9 Add corpus taxonomy tooling 2026-03-10 f06fb75 Extract the internal line-break core 2026-03-10 614dfab Add a line-by-line reflow API and contour demo 2026-03-10 d782b14 Add explicit locale control for text analysis 2026-03-10 a734f17 Expose discretionary hyphen state to userland renderers 2026-03-10 c4cf982 Split prepare internals and add a second Japanese canary 2026-03-10 da60460 Split benchmark prepare timings by phase 2026-03-10 e30c04e Extract bidi logic from layout 2026-03-10 830d8ad Add a Japanese prose canary and fix iteration marks 2026-03-10 93b10a6 Keep bidi metadata off the fast prepare path 2026-03-10 c94b2ba Improve SHY diagnostics and line rendering 2026-03-10 b89a7fb Make layout imports safe outside the DOM 2026-03-10 b6e2104 Finish the Myanmar phrase-glue canary 2026-03-10 c9cbfca Tighten Myanmar punctuation glue and probe diagnostics 2026-03-10 5e92358 Make prepare() return an opaque handle 2026-03-10 86f741f Refine mixed app-text breaking and diagnostics 2026-03-10 7995c72 Add a mixed app-text canary and glue URL runs 2026-03-10 d9f9873 Model discretionary soft hyphen breaks 2026-03-10 c340d09 Cache richer segment metrics during prepare 2026-03-10 62b258a Strengthen break semantics beyond plain spaces 2026-03-09 d988bb0 Keep layout on a specialized fast path 2026-03-09 3e991df Share line breaking between layout APIs 2026-03-09 4715123 Add Thai stress corpus and fix ASCII quote wrapping 2026-03-09 49e0f36 Loosen non-Safari edge tolerance for remaining Arabic fits 2026-03-09 8aaf7b8 Keep Arabic punctuation-plus-mark clusters with their word 2026-03-09 9612485 Refine Arabic preprocessing around punctuation and marks 2026-03-09 d91a0af Merge Arabic no-space punctuation clusters 2026-03-08 7fb726d Fingerprint browser test environments and fix Korean quote wraps in Chromium 2026-03-08 16c04fe Improve Arabic and Hindi corpus wrapping 2026-03-08 c852293 Simplify merged segmentation and CJK measurement 2026-03-08 06e6437 Keep apostrophe-led elisions with the following word 2026-03-08 3f9cb1e Separate prepare analysis from measurement 2026-03-07 8f4ac00 Harden browser-accuracy diagnostics without slowing layout 2026-03-07 9af7f63 Close the remaining browser accuracy gaps cheaply 2026-03-07 5cd5762 Add Gatsby diagnostics without regressing hot-path layout 2026-03-07 d07dd7a gatsby test: first pass 2026-03-07 89ed442 Document public layout data fields 2026-03-06 7a4f827 Make lineHeight an explicit layout-time input 2026-03-06 0aca7b6 Link system-ui warnings to RESEARCH.md exploration 2026-03-06 fea1fa7 Split fast and rich layout APIs 2026-03-06 dbebb7d Clean up stale docs and unify test data 2026-03-04 5ecce72 Simplify layout code: flatten data, remove dead computation, cut allocations 2026-03-04 76c2766 Update all docs for 4-font accuracy, clean up stale comments 2026-03-04 bad8fbc CJK-split all CJK segments, not just word-like — Firefox 99.7% → 99.9% 2026-03-04 573b6db Fix emoji correction for Safari — compare canvas vs DOM, not fontSize 2026-03-03 e2f1321 Document char-level experiment, prior art, fix text-layout repo URL 2026-03-03 fee6750 Opening bracket forward-merge, CSS config docs, Chrome 3838/3840 2026-03-03 aff5ba7 Benchmark CJK scaling, cleanup for...of, --no-hmr, NaN guard 2026-03-03 ae466fa Hoist word segmenter to module level, add TODO section 2026-03-03 e8de24c Fix prepare() perf regression: skip emoji grapheme scan for non-emoji text 2026-03-03 315658b Document cache design decision, slim down CLAUDE.md 2026-03-03 6ce9982 Kinsoku shori — CJK line-breaking rules, Chrome 99.7% → 99.9% 2026-03-03 91304c0 Emoji correction, merging fix, overflow handling — Chrome 99.4% → 99.7% 2026-03-03 f54a344 Revert trailing space experiment — restore Chrome 99.4% 2026-03-03 73783d0 Add line-break diagnostic, document Safari CSS behavior differences 2026-03-03 820e8e1 Add comments documenting motivations throughout 2026-03-03 f962f72 Initial project setupa7:T1f88,Command: /bin/zsh -lc "sed -n '1,260p' pages/benchmark.ts" Chunk ID: 9ba28d Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1977 Output: import { prepare, prepareWithSegments, layout, layoutNextLine, layoutWithLines, walkLineRanges, clearCache, profilePrepare, } from '../src/layout.ts' import type { PreparedText, PreparedTextWithSegments } from '../src/layout.ts' import { TEXTS } from '../src/test-data.ts' import { clearNavigationReport, publishNavigationPhase, publishNavigationReport as publishHashReport, } from './report-utils.ts' import arRisalatAlGhufranPart1 from '../corpora/ar-risalat-al-ghufran-part-1.txt' with { type: 'text' } import hiEidgah from '../corpora/hi-eidgah.txt' with { type: 'text' } import jaKumoNoIto from '../corpora/ja-kumo-no-ito.txt' with { type: 'text' } import jaRashomon from '../corpora/ja-rashomon.txt' with { type: 'text' } import kmPrachumReuangPrengKhmerVolume7Stories1To10 from '../corpora/km-prachum-reuang-preng-khmer-volume-7-stories-1-10.txt' with { type: 'text' } import myBadDeedsReturnToYouTeacher from '../corpora/my-bad-deeds-return-to-you-teacher.txt' with { type: 'text' } import myCunningHeronTeacher from '../corpora/my-cunning-heron-teacher.txt' with { type: 'text' } import koUnsuJohEunNal from '../corpora/ko-unsu-joh-eun-nal.txt' with { type: 'text' } import thNithanVetalStory1 from '../corpora/th-nithan-vetal-story-1.txt' with { type: 'text' } import urChughd from '../corpora/ur-chughd.txt' with { type: 'text' } import zhGuxiang from '../corpora/zh-guxiang.txt' with { type: 'text' } import zhZhufu from '../corpora/zh-zhufu.txt' with { type: 'text' } const COUNT = 500 const FONT_FAMILY = '"Helvetica Neue", Helvetica, Arial, sans-serif' const FONT_SIZE = 16 const FONT = `${FONT_SIZE}px ${FONT_FAMILY}` const LINE_HEIGHT = Math.round(FONT_SIZE * 1.2) const WIDTH_BEFORE = 400 const WIDTH_AFTER = 300 const WARMUP = 2 const RUNS = 10 const PREPARE_SAMPLE_REPEATS = 1 const LAYOUT_SAMPLE_REPEATS = 200 const LAYOUT_SAMPLE_WIDTHS = [200, 250, 300, 350, 400] as const const DOM_BATCH_SAMPLE_REPEATS = 1 const DOM_INTERLEAVED_SAMPLE_REPEATS = 1 const RICH_COUNT = 60 const RICH_LAYOUT_SAMPLE_REPEATS = 40 const RICH_LAYOUT_SAMPLE_WIDTHS = [180, 220, 260] as const const RICH_LONG_REPEAT = 8 const RICH_LONG_SAMPLE_WIDTHS = [240, 300, 360] as const const CORPUS_LAYOUT_SAMPLE_REPEATS = 200 const CORPUS_WARMUP = 1 const CORPUS_RUNS = 7 type BenchmarkResult = { label: string, ms: number, desc: string } type CorpusBenchmarkResult = { id: string label: string font: string chars: number analysisSegments: number segments: number breakableSegments: number width: number lineCount: number analysisMs: number measureMs: number prepareMs: number layoutMs: number } type BenchmarkReport = { status: 'ready' | 'error' requestId?: string results?: BenchmarkResult[] richResults?: BenchmarkResult[] richLongResults?: BenchmarkResult[] corpusResults?: CorpusBenchmarkResult[] message?: string } const params = new URLSearchParams(location.search) const reportMode = params.get('report') === '1' const requestId = params.get('requestId') ?? undefined const CORPORA = [ { id: 'ja-kumo-no-ito', label: 'Japanese prose (story 2)', text: jaKumoNoIto, font: '20px "Hiragino Mincho ProN", "Yu Mincho", "Noto Serif CJK JP", serif', lineHeight: 32, width: 300, sampleWidths: [240, 300, 360] as const, }, { id: 'ja-rashomon', label: 'Japanese prose', text: jaRashomon, font: '20px "Hiragino Mincho ProN", "Yu Mincho", "Noto Serif CJK JP", serif', lineHeight: 32, width: 300, sampleWidths: [240, 300, 360] as const, }, { id: 'ko-unsu-joh-eun-nal', label: 'Korean prose', text: koUnsuJohEunNal, font: '18px "Apple SD Gothic Neo", "Malgun Gothic", "Noto Sans CJK KR", sans-serif', lineHeight: 30, width: 300, sampleWidths: [240, 300, 360] as const, }, { id: 'zh-zhufu', label: 'Chinese prose', text: zhZhufu, font: '20px "Songti SC", "PingFang SC", "Noto Serif CJK SC", serif', lineHeight: 32, width: 300, sampleWidths: [240, 300, 360] as const, }, { id: 'zh-guxiang', label: 'Chinese prose (story 2)', text: zhGuxiang, font: '20px "Songti SC", "PingFang SC", "Noto Serif CJK SC", serif', lineHeight: 32, width: 300, sampleWidths: [240, 300, 360] as const, }, { id: 'th-nithan-vetal-story-1', label: 'Thai prose', text: thNithanVetalStory1, font: '20px "Thonburi", "Noto Sans Thai", sans-serif', lineHeight: 32, width: 300, sampleWidths: [240, 300, 360] as const, }, { id: 'my-cunning-heron-teacher', label: 'Myanmar prose', text: myCunningHeronTeacher, font: '20px "Myanmar MN", "Myanmar Sangam MN", "Noto Sans Myanmar", serif', lineHeight: 32, width: 300, sampleWidths: [240, 300, 360] as const, }, { id: 'my-bad-deeds-return-to-you-teacher', label: 'Myanmar prose (story 2)', text: myBadDeedsReturnToYouTeacher, font: '20px "Myanmar MN", "Myanmar Sangam MN", "Noto Sans Myanmar", serif', lineHeight: 32, width: 300, sampleWidths: [240, 300, 360] as const, }, { id: 'ur-chughd', label: 'Urdu prose', text: urChughd, font: '20px "Noto Nastaliq Urdu", "DecoType Nastaleeq Urdu UI", "Geeza Pro", serif', lineHeight: 38, width: 300, sampleWidths: [240, 300, 360] as const, }, { id: 'km-prachum-reuang-preng-khmer-volume-7-stories-1-10', label: 'Khmer prose', text: kmPrachumReuangPrengKhmerVolume7Stories1To10, font: '20px "Khmer Sangam MN", "Khmer MN", "Noto Sans Khmer", serif', lineHeight: 32, width: 300, sampleWidths: [240, 300, 360] as const, }, { id: 'hi-eidgah', label: 'Hindi prose', text: hiEidgah, font: '20px "Kohinoor Devanagari", "Noto Serif Devanagari", serif', lineHeight: 32, width: 300, sampleWidths: [240, 300, 360] as const, }, { id: 'ar-risalat-al-ghufran-part-1', label: 'Arabic prose', text: arRisalatAlGhufranPart1, font: '20px "Geeza Pro", "Noto Naskh Arabic", "Arial", serif', lineHeight: 34, width: 300, sampleWidths: [240, 300, 360] as const, }, ] as const // Filter edge cases — not realistic comments const commentTexts = TEXTS.filter(t => t.text.trim().length > 1) const texts: string[] = [] for (let i = 0; i < COUNT; i++) { texts.push(commentTexts[i % commentTexts.length]!.text) } declare global { interface Window { __BENCHMARK_REPORT__?: BenchmarkReport } } function median(times: number[]): number { const sorted = [...times].sort((a, b) => a - b) const mid = Math.floor(sorted.length / 2) return sorted.length % 2 === 0 ? (sorted[mid - 1]! + sorted[mid]!) / 2 : sorted[mid]! } function bench( fn: (repeatIndex: number) => void, sampleRepeats = 1, warmup = WARMUP, runs = RUNS, ): number { function runRepeated(): void { for (let r = 0; r < sampleRepeats; r++) { fn(r) } } for (let i = 0; i < warmup; i++) runRepeated() const times: number[] = [] for (let i = 0; i < runs; i++) { const t0 = performance.now() runRepeated() times.push((performance.now() - t0) / sampleRepeats) } return median(times) } // Yield to let the browser paint status updates function nextFrame(): Promise { return new Promise(resolve => { requestAnimationFrame(() => { resolve() }) }) } function withRequestId(report: T): BenchmarkReport { return requestId === undefined ? report : { ...report, requestId } } function publishNavigationReport(report: BenchmarkReport): void { if (!reportMode) return publishHashReport(report) } function setReport(report: BenchmarkReport): void { window.__BENCHMARK_REPORT__ = report publishNavigationReport(report) } function buildCorpusBenchmarks(): CorpusBenchmarkResult[] { const corpusResults: CorpusBenchmarkResult[] = [] let corpusLayoutSink = 0 for (const corpus of CORPORA) { a8:T1eca,Command: /bin/zsh -lc "sed -n '1,260p' pages/accuracy.ts" Chunk ID: cf9d46 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1930 Output: import { clearCache, layout, layoutWithLines, prepareWithSegments, type PreparedTextWithSegments, } from '../src/layout.ts' import { getDiagnosticUnits } from './diagnostic-utils.ts' import { clearNavigationReport, publishNavigationPhase, publishNavigationReport } from './report-utils.ts' import { TEXTS, SIZES, WIDTHS } from '../src/test-data.ts' const FONTS = [ '"Helvetica Neue", Helvetica, Arial, sans-serif', 'Georgia, "Times New Roman", serif', 'Verdana, Geneva, sans-serif', '"Courier New", Courier, monospace', ] type Mismatch = { label: string font: string fontSize: number lineHeight: number width: number actual: number predicted: number diff: number text: string diagnosticLines?: string[] } type AccuracyRow = { label: string font: string fontSize: number lineHeight: number width: number actual: number predicted: number diff: number } type AccuracyReport = { status: 'ready' | 'error' requestId?: string environment?: EnvironmentFingerprint total?: number matchCount?: number mismatchCount?: number mismatches?: Mismatch[] rows?: AccuracyRow[] message?: string } type AccuracyNavigationReport = { status: 'ready' | 'error' requestId?: string total?: number matchCount?: number mismatchCount?: number message?: string } type EnvironmentFingerprint = { userAgent: string devicePixelRatio: number viewport: { innerWidth: number innerHeight: number outerWidth: number outerHeight: number visualViewportScale: number | null } screen: { width: number height: number availWidth: number availHeight: number colorDepth: number pixelDepth: number } } declare global { interface Window { __ACCURACY_REPORT__?: AccuracyReport } } const params = new URLSearchParams(location.search) const requestId = params.get('requestId') ?? undefined const includeFullRows = params.get('full') === '1' const reportEndpoint = params.get('reportEndpoint') function withRequestId(report: T): AccuracyReport { return requestId === undefined ? report : { ...report, requestId } } function getEnvironmentFingerprint(): EnvironmentFingerprint { return { userAgent: navigator.userAgent, devicePixelRatio: window.devicePixelRatio, viewport: { innerWidth: window.innerWidth, innerHeight: window.innerHeight, outerWidth: window.outerWidth, outerHeight: window.outerHeight, visualViewportScale: window.visualViewport?.scale ?? null, }, screen: { width: window.screen.width, height: window.screen.height, availWidth: window.screen.availWidth, availHeight: window.screen.availHeight, colorDepth: window.screen.colorDepth, pixelDepth: window.screen.pixelDepth, }, } } function publishReport(report: AccuracyReport): void { const reportJson = JSON.stringify(report) window.__ACCURACY_REPORT__ = report if (reportEndpoint !== null) { publishNavigationPhase('posting', requestId) void (async () => { try { await fetch(reportEndpoint, { method: 'POST', body: reportJson, }) publishNavigationReport(toNavigationReport(report)) } catch { // Best-effort side channel for large reports. } })() return } publishNavigationReport(toNavigationReport(report)) } function toNavigationReport(report: AccuracyReport): AccuracyNavigationReport { if (report.status === 'error') { return { status: report.status, ...(report.requestId === undefined ? {} : { requestId: report.requestId }), ...(report.message === undefined ? {} : { message: report.message }), } } return { status: report.status, ...(report.requestId === undefined ? {} : { requestId: report.requestId }), ...(report.total === undefined ? {} : { total: report.total }), ...(report.matchCount === undefined ? {} : { matchCount: report.matchCount }), ...(report.mismatchCount === undefined ? {} : { mismatchCount: report.mismatchCount }), } } function getBrowserLines( prepared: PreparedTextWithSegments, div: HTMLDivElement, ): string[] { const textNode = div.firstChild if (!(textNode instanceof Text)) return [] const units = getDiagnosticUnits(prepared) const range = document.createRange() const browserLines: string[] = [] let currentLine = '' let lastTop: number | null = null for (const unit of units) { range.setStart(textNode, unit.start) range.setEnd(textNode, unit.end) const rects = range.getClientRects() const rectTop: number | null = rects.length > 0 ? rects[0]!.top : lastTop if (rectTop !== null && lastTop !== null && rectTop > lastTop + 0.5) { browserLines.push(currentLine) currentLine = unit.text } else { currentLine += unit.text } if (rectTop !== null) lastTop = rectTop } if (currentLine) browserLines.push(currentLine) return browserLines } function runSweep(): { total: number, mismatches: Mismatch[], rows: AccuracyRow[] } { const container = document.createElement('div') container.style.cssText = 'position:absolute;top:-9999px;left:-9999px;visibility:hidden' document.body.appendChild(container) const mismatches: Mismatch[] = [] const rows: AccuracyRow[] = [] let total = 0 for (const fontFamily of FONTS) { for (const fontSize of SIZES) { const font = `${fontSize}px ${fontFamily}` const lineHeight = Math.round(fontSize * 1.2) clearCache() for (const maxWidth of WIDTHS) { const divs: HTMLDivElement[] = [] const prepared: PreparedTextWithSegments[] = [] for (const { text } of TEXTS) { const div = document.createElement('div') div.style.font = font div.style.lineHeight = `${lineHeight}px` div.style.width = `${maxWidth}px` div.style.wordWrap = 'break-word' div.style.overflowWrap = 'break-word' div.textContent = text container.appendChild(div) divs.push(div) prepared.push(prepareWithSegments(text, font)) } for (let i = 0; i < TEXTS.length; i++) { const { label, text } = TEXTS[i]! const actual = divs[i]!.getBoundingClientRect().height const predicted = layout(prepared[i]!, maxWidth, lineHeight).height rows.push({ label, font: fontFamily, fontSize, lineHeight, width: maxWidth, actual, predicted, diff: predicted - actual, }) total++ if (Math.abs(actual - predicted) >= 1) { const browserLines = getBrowserLines(prepared[i]!, divs[i]!) const ourLayout = layoutWithLines(prepared[i]!, maxWidth, lineHeight) const lineDetails: string[] = [] const maxLines = Math.max(browserLines.length, ourLayout.lines.length) for (let li = 0; li < maxLines; li++) { const ours = (ourLayout.lines[li]?.text ?? '').trimEnd() const theirs = (browserLines[li] ?? '').trimEnd() if (ours !== theirs) { lineDetails.push(`L${li+1} ours="${ours.slice(0,40)}" browser="${theirs.slice(0,40)}"`) } } if (lineDetails.length === 0 && browserLines.length !== ourLayout.lines.length) { lineDetails.push(`ours=${ourLayout.lines.length}L browser=${browserLines.length}L (same content, different count?)`) } mismatches.push({ label, font: fontFamily, fontSize, lineHeight, a9:T943,Command: /bin/zsh -lc "sed -n '1,260p' pages/diagnostic-utils.ts" Chunk ID: b98ae5 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 551 Output: import type { PreparedTextWithSegments } from '../src/layout.ts' export type DiagnosticUnit = { text: string start: number end: number } const diagnosticGraphemeSegmenter = new Intl.Segmenter(undefined, { granularity: 'grapheme' }) export function getDiagnosticUnits(prepared: PreparedTextWithSegments): DiagnosticUnit[] { const units: DiagnosticUnit[] = [] let offset = 0 for (let i = 0; i < prepared.segments.length; i++) { const segment = prepared.segments[i]! if (prepared.breakableWidths[i] !== null) { let localOffset = 0 for (const grapheme of diagnosticGraphemeSegmenter.segment(segment)) { const start = offset + localOffset localOffset += grapheme.segment.length units.push({ text: grapheme.segment, start, end: offset + localOffset }) } } else { units.push({ text: segment, start: offset, end: offset + segment.length }) } offset += segment.length } return units } export function getLineContent(text: string, end: number): { text: string, end: number } { const trimmed = text.trimEnd() return { text: trimmed, end: end - (text.length - trimmed.length), } } export function formatBreakContext(text: string, breakOffset: number, radius = 32): string { const start = Math.max(0, breakOffset - radius) const end = Math.min(text.length, breakOffset + radius) return `${start > 0 ? '…' : ''}${text.slice(start, breakOffset)}|${text.slice(breakOffset, end)}${end < text.length ? '…' : ''}` } export function measureCanvasTextWidth( ctx: CanvasRenderingContext2D, text: string, font: string, ): number { ctx.font = font return ctx.measureText(text).width } export function measureDomTextWidth( doc: Document, text: string, font: string, direction: string, ): number { const span = doc.createElement('span') span.style.position = 'absolute' span.style.visibility = 'hidden' span.style.whiteSpace = 'pre' span.style.font = font span.style.direction = direction span.style.unicodeBidi = 'plaintext' span.textContent = text doc.body.appendChild(span) const width = span.getBoundingClientRect().width doc.body.removeChild(span) return width } aa:T1af5,Command: /bin/zsh -lc "sed -n '1,260p' scripts/browser-automation.ts" Chunk ID: 78bd98 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1682 Output: import { execFileSync, spawn, type ChildProcess } from 'node:child_process' import { closeSync, mkdirSync, mkdtempSync, openSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { createConnection, createServer as createNetServer } from 'node:net' import { tmpdir } from 'node:os' import { join } from 'node:path' import { readNavigationPhaseState, readNavigationReportText, type NavigationPhase } from '../shared/navigation-state.ts' export type BrowserKind = 'chrome' | 'safari' | 'firefox' export type AutomationBrowserKind = BrowserKind type MaybePromise = T | Promise export type BrowserSession = { navigate: (url: string) => MaybePromise readLocationUrl: () => MaybePromise close: () => void } export type BrowserSessionOptions = { foreground?: boolean } export type PageServer = { baseUrl: string process: ChildProcess | null } export type BrowserAutomationLock = { release: () => void } function runAppleScript(lines: string[]): string { return execFileSync( 'osascript', lines.flatMap(line => ['-e', line]), { encoding: 'utf8' }, ).trim() } function getFrontmostApplicationName(): string | null { try { return runAppleScript([ 'tell application "System Events"', 'return name of first application process whose frontmost is true', 'end tell', ]) } catch { return null } } function restoreFrontmostApplication(name: string | null): void { if (name === null || name.length === 0) return try { runAppleScript([`tell application ${JSON.stringify(name)} to activate`]) } catch { // Best effort restore only. } } function runBackgroundAppleScript(lines: string[]): string { const frontmost = getFrontmostApplicationName() try { return runAppleScript(lines) } finally { restoreFrontmostApplication(frontmost) } } export function sleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)) } async function waitForPort(port: number): Promise { for (let i = 0; i < 200; i++) { const open = await new Promise(resolve => { const socket = createConnection({ host: '127.0.0.1', port }) let settled = false const finish = (value: boolean): void => { if (settled) return settled = true socket.destroy() resolve(value) } socket.once('connect', () => finish(true)) socket.once('error', () => finish(false)) }) if (open) return await sleep(100) } throw new Error(`Timed out waiting for local port ${port}`) } export async function getAvailablePort(requestedPort: number | null = null): Promise { if (requestedPort !== null && Number.isFinite(requestedPort) && requestedPort > 0) { return requestedPort } return await new Promise((resolve, reject) => { const server = createNetServer() server.once('error', reject) server.listen(0, '127.0.0.1', () => { const address = server.address() if (address === null || typeof address === 'string') { reject(new Error('Failed to allocate a free port')) return } const { port } = address server.close(error => { if (error) { reject(error) return } resolve(port) }) }) }) } const LOCK_DIR = join(process.env['TMPDIR'] ?? '/tmp', 'pretext-browser-automation-locks') type LockMetadata = { pid: number startedAt: number } function readLockMetadata(lockPath: string): LockMetadata | null { try { const raw = readFileSync(lockPath, 'utf8') const parsed = JSON.parse(raw) as Partial if ( typeof parsed.pid !== 'number' || !Number.isInteger(parsed.pid) || parsed.pid <= 0 || typeof parsed.startedAt !== 'number' ) { return null } return { pid: parsed.pid, startedAt: parsed.startedAt, } } catch { return null } } function isProcessAlive(pid: number): boolean { try { process.kill(pid, 0) return true } catch (error) { if ( error instanceof Error && 'code' in error && error.code === 'EPERM' ) { return true } return false } } export async function acquireBrowserAutomationLock( browser: AutomationBrowserKind, timeoutMs = 120_000, ): Promise { mkdirSync(LOCK_DIR, { recursive: true }) const lockPath = join(LOCK_DIR, `${browser}.lock`) const start = Date.now() while (true) { try { const fd = openSync(lockPath, 'wx') writeFileSync(fd, JSON.stringify({ pid: process.pid, startedAt: Date.now(), })) let released = false return { release() { if (released) return released = true try { closeSync(fd) } catch { // Ignore close races during teardown. } try { rmSync(lockPath) } catch { // Best effort cleanup. } }, } } catch (error) { if (!(error instanceof Error) || !String(error).includes('EEXIST')) throw error const metadata = readLockMetadata(lockPath) if (metadata !== null && !isProcessAlive(metadata.pid)) { try { rmSync(lockPath) continue } catch { // Another process may have replaced or removed it. Retry normally. } } if (Date.now() - start >= timeoutMs) { throw new Error(`Timed out waiting for ${browser} automation lock`) } await sleep(250) } } } async function canReachUrl(url: string): Promise { try { const response = await fetch(url) return response.ok } catch { return false } } const LOOPBACK_BASES = [ 'http://127.0.0.1', 'http://localhost', 'http://[::1]', ] async function resolveBaseUrl(port: number, pathname: string): Promise { for (const base of LOOPBACK_BASES) { const url = `${base}:${port}${pathname}` if (await canReachUrl(url)) { return `${base}:${port}` } } return null } function getTimeoutMessage( browser: BrowserKind, target: 'report' | 'posted report', lastPhase: NavigationPhase | null, ): string { if (lastPhase === null) { return `Timed out waiting for ${target} from ${browser}` } return `Timed out waiting for ${target} from ${browser} (last phase: ${lastPhase})` } async function readLastNavigationPhase( session: BrowserSession, expectedRequestId: string, ): Promise { const currentUrl = await session.readLocationUrl() const phaseState = readNavigationPhaseState(currentUrl) if (phaseState === null) return null ab:T1f24,Command: /bin/zsh -lc "sed -n '261,520p' scripts/browser-automation.ts" Chunk ID: fd09bc Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1949 Output: if (phaseState.requestId !== undefined && phaseState.requestId !== expectedRequestId) { return null } return phaseState.phase } type BidiResponse = { id: number result?: unknown error?: string message?: string type?: string } type FirefoxBidiClient = { send: (method: string, params?: Record) => Promise close: () => void } type FirefoxSessionState = { bidi: FirefoxBidiClient context: string firefoxProcess: ChildProcess profileDir: string } async function connectFirefoxBidi(port: number): Promise { const ws = new WebSocket(`ws://127.0.0.1:${port}/session`) const pending = new Map void>() let nextId = 1 ws.onmessage = event => { const message = JSON.parse(String(event.data)) as BidiResponse if (message.id === undefined) return const resolve = pending.get(message.id) if (resolve !== undefined) { pending.delete(message.id) resolve(message) } } await new Promise((resolve, reject) => { ws.onopen = () => resolve() ws.onerror = event => reject(new Error(String((event as ErrorEvent).message ?? 'Firefox WebSocket error'))) }) return { async send(method: string, params: Record = {}): Promise { const id = nextId++ ws.send(JSON.stringify({ id, method, params })) return await new Promise(resolve => pending.set(id, resolve)) }, close() { ws.close() }, } } function getBidiStringValue(response: BidiResponse): string { const remoteResult = response.result as { type?: string result?: { type?: string value?: unknown } } | undefined const value = remoteResult?.result?.value return typeof value === 'string' ? value : '' } function closeFirefoxSessionState(state: FirefoxSessionState): void { state.bidi.close() try { state.firefoxProcess.kill('SIGTERM') } catch { // Best effort cleanup. } rmSync(state.profileDir, { recursive: true, force: true }) } async function initializeFirefoxSession(): Promise { const bidiPort = await getAvailablePort() const profileDir = mkdtempSync(join(tmpdir(), 'pretext-firefox-')) const firefoxProcess = spawn('/Applications/Firefox.app/Contents/MacOS/firefox', [ '--headless', '--new-instance', '--profile', profileDir, '--remote-debugging-port', String(bidiPort), 'about:blank', ], { cwd: process.cwd(), stdio: 'ignore', }) let bidi: FirefoxBidiClient | null = null try { await waitForPort(bidiPort) bidi = await connectFirefoxBidi(bidiPort) const session = await bidi.send('session.new', { capabilities: { alwaysMatch: {} } }) if (session.error !== undefined) { throw new Error(session.message ?? session.error) } const tree = await bidi.send('browsingContext.getTree', {}) if (tree.error !== undefined) { throw new Error(tree.message ?? tree.error) } const contexts = (tree.result as { contexts: Array<{ context: string }> }).contexts const context = contexts[0]?.context if (context === undefined) { throw new Error('Firefox BiDi returned no browsing context') } return { bidi, context, firefoxProcess, profileDir, } } catch (error) { bidi?.close() try { firefoxProcess.kill('SIGTERM') } catch { // Best effort cleanup. } rmSync(profileDir, { recursive: true, force: true }) throw error } } function createSafariSession(options: BrowserSessionOptions): BrowserSession { const scriptLines = ['tell application "Safari"'] if (options.foreground === true) { scriptLines.unshift('tell application "Safari" to activate') } scriptLines.push('set targetDocument to make new document with properties {URL:"about:blank"}') if (options.foreground === true) { scriptLines.push('set targetWindow to front window') scriptLines.push('set index of targetWindow to 1') } scriptLines.push('return id of front window as string') scriptLines.push('end tell') const windowIdRaw = options.foreground === true ? runAppleScript(scriptLines) : runBackgroundAppleScript(scriptLines) const windowId = Number.parseInt(windowIdRaw, 10) if (!Number.isFinite(windowId)) { throw new Error(`Failed to create Safari automation window: ${windowIdRaw}`) } return { navigate(url) { const navigateLines = [ 'tell application "Safari"', `set targetWindow to first window whose id is ${windowId}`, ] if (options.foreground === true) { navigateLines.unshift('tell application "Safari" to activate') navigateLines.push('set index of targetWindow to 1') } navigateLines.push(`set URL of current tab of targetWindow to ${JSON.stringify(url)}`) navigateLines.push('end tell') if (options.foreground === true) { runAppleScript(navigateLines) } else { runBackgroundAppleScript(navigateLines) } }, readLocationUrl() { try { return runAppleScript([ 'tell application "Safari"', `return URL of current tab of (first window whose id is ${windowId})`, 'end tell', ]) } catch { return '' } }, close() { try { runAppleScript([ 'tell application "Safari"', `close (first window whose id is ${windowId})`, 'end tell', ]) } catch { // Ignore cleanup failures if the user already closed the window. } }, } } function createChromeSession(options: BrowserSessionOptions): BrowserSession { const scriptLines = [ 'tell application "Google Chrome"', 'if (count of windows) = 0 then make new window', 'set targetWindow to front window', 'set targetTab to make new tab at end of tabs of targetWindow with properties {URL:"about:blank"}', ] if (options.foreground === true) { scriptLines.splice(1, 0, 'activate') scriptLines.push('set active tab index of targetWindow to (count of tabs of targetWindow)') } scriptLines.push('return (id of targetWindow as string) & "," & (id of targetTab as string)') scriptLines.push('end tell') const identifiers = options.foreground === true ? runAppleScript(scriptLines) : runBackgroundAppleScript(scriptLines) const [windowIdRaw, tabIdRaw] = identifiers.split(',') const windowId = Number.parseInt(windowIdRaw ?? '', 10) const tabId = Number.parseInt(tabIdRaw ?? '', 10) if (!Number.isFinite(windowId) || !Number.isFinite(tabId)) { throw new Error(`Failed to create Chrome automation tab: ${identifiers}`) } return { navigate(url) { const navigateLines = [ 'tell application "Google Chrome"', `set targetWindow to first window whose id is ${windowId}`, `set URL of (first tab of targetWindow whose id is ${tabId}) to ${JSON.stringify(url)}`, 'end tell', ] if (options.foreground === true) { runAppleScript(navigateLines) } else { runBackgroundAppleScript(navigateLines) } }, readLocationUrl() { try { return runAppleScript([ 'tell application "Google Chrome"', `set targetWindow to first window whose id is ${windowId}`, `return URL of (first tab of targetWindow whose id is ${tabId})`, 'end tell', ]) } catch { return '' } }, close() { try { runAppleScript([ 'tell application "Google Chrome"', `set targetWindow to first window whose id is ${windowId}`, `close (first tab of targetWindow whose id is ${tabId})`, 'end tell', ac:T22bd,Command: /bin/zsh -lc "sed -n '1,260p' scripts/corpus-check.ts" Chunk ID: 580d14 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 2181 Output: import { type ChildProcess } from 'node:child_process' import { acquireBrowserAutomationLock, createBrowserSession, ensurePageServer, getAvailablePort, loadHashReport, type BrowserKind, } from './browser-automation.ts' type CorpusMeta = { id: string language: string title: string min_width?: number max_width?: number default_width?: number } type CorpusOverrideOptions = { font: string | null lineHeight: number | null method: 'span' | 'range' | null sliceStart: number | null sliceEnd: number | null } type CorpusReport = { status: 'ready' | 'error' requestId?: string environment?: { userAgent: string devicePixelRatio: number viewport: { innerWidth: number innerHeight: number outerWidth: number outerHeight: number visualViewportScale: number | null } screen: { width: number height: number availWidth: number availHeight: number colorDepth: number pixelDepth: number } } corpusId?: string sliceStart?: number | null sliceEnd?: number | null title?: string width?: number predictedHeight?: number actualHeight?: number diffPx?: number predictedLineCount?: number browserLineCount?: number browserLineMethod?: 'span-probe' | 'range' alternateBrowserLineMethod?: 'span-probe' | 'range' alternateBrowserLineCount?: number probeHeight?: number normalizedHeight?: number mismatchCount?: number firstMismatch?: { line: number ours: string browser: string } | null firstBreakMismatch?: { line: number oursStart: number browserStart: number oursEnd: number browserEnd: number oursText: string browserText: string oursRenderedText: string browserRenderedText: string oursContext: string browserContext: string deltaText: string reasonGuess: string oursSumWidth: number oursDomWidth: number oursFullWidth: number browserDomWidth: number browserFullWidth: number oursSegments: Array<{ text: string width: number domWidth: number isSpace: boolean }> } | null alternateFirstBreakMismatch?: object | null extractorSensitivity?: string | null maxLineWidthDrift?: number maxDriftLine?: { line: number drift: number text: string sumWidth: number fullWidth: number domWidth: number pairAdjustedWidth: number segments: Array<{ text: string width: number domWidth: number isSpace: boolean }> } | null message?: string } function parseStringFlag(name: string): string | null { const prefix = `--${name}=` const arg = process.argv.find(value => value.startsWith(prefix)) return arg === undefined ? null : arg.slice(prefix.length) } function parseNumberFlag(name: string, fallback: number): number { const raw = parseStringFlag(name) if (raw === null) return fallback const parsed = Number.parseInt(raw, 10) if (!Number.isFinite(parsed)) { throw new Error(`Invalid value for --${name}: ${raw}`) } return parsed } function parseBrowser(value: string | null): BrowserKind { const browser = (value ?? process.env['CORPUS_CHECK_BROWSER'] ?? 'chrome').toLowerCase() if (browser !== 'chrome' && browser !== 'safari') { throw new Error(`Unsupported browser ${browser}; expected chrome or safari`) } return browser } function parseOptionalNumberFlag(name: string): number | null { const raw = parseStringFlag(name) if (raw === null) return null const parsed = Number.parseInt(raw, 10) if (!Number.isFinite(parsed)) { throw new Error(`Invalid value for --${name}: ${raw}`) } return parsed } function hasFlag(name: string): boolean { return process.argv.includes(`--${name}`) } async function loadSources(): Promise { return await Bun.file('corpora/sources.json').json() } function getTargetWidths(meta: CorpusMeta): number[] { const widths = process.argv.slice(2) .filter(arg => !arg.startsWith('--')) .map(arg => Number.parseInt(arg, 10)) .filter(width => Number.isFinite(width)) if (widths.length > 0) return widths const min = meta.min_width ?? 300 const max = meta.max_width ?? 900 const preferred = [min, Math.max(min, Math.min(max, meta.default_width ?? 600)), max] return [...new Set(preferred)] } function appendOverrideParams(url: string, overrides: CorpusOverrideOptions): string { let nextUrl = url if (overrides.font !== null) { nextUrl += `&font=${encodeURIComponent(overrides.font)}` } if (overrides.lineHeight !== null) { nextUrl += `&lineHeight=${overrides.lineHeight}` } if (overrides.method !== null) { nextUrl += `&method=${encodeURIComponent(overrides.method)}` } if (overrides.sliceStart !== null) { nextUrl += `&sliceStart=${overrides.sliceStart}` } if (overrides.sliceEnd !== null) { nextUrl += `&sliceEnd=${overrides.sliceEnd}` } return nextUrl } function printReport(report: CorpusReport): void { if (report.status === 'error') { console.log(`error: ${report.message ?? 'unknown error'}`) return } const width = report.width ?? 0 const diff = Math.round(report.diffPx ?? 0) const predicted = Math.round(report.predictedHeight ?? 0) const actual = Math.round(report.actualHeight ?? 0) const lines = report.predictedLineCount !== undefined && report.browserLineCount !== undefined ? `${report.predictedLineCount}/${report.browserLineCount}` : '-' console.log( `width ${width}: diff ${diff > 0 ? '+' : ''}${diff}px | height ${predicted}/${actual} | lines ${lines}`, ) if (report.sliceStart !== undefined || report.sliceEnd !== undefined) { console.log(` slice: ${report.sliceStart ?? 0}-${report.sliceEnd ?? '-'}`) } if (report.maxLineWidthDrift !== undefined) { console.log(` max line width drift: ${report.maxLineWidthDrift.toFixed(3)}px`) } if (report.environment !== undefined) { const env = report.environment console.log( ` env: dpr ${env.devicePixelRatio} | viewport ${env.viewport.innerWidth}x${env.viewport.innerHeight} | outer ${env.viewport.outerWidth}x${env.viewport.outerHeight} | scale ${env.viewport.visualViewportScale ?? '-'} | screen ${env.screen.width}x${env.screen.height}`, ) } if (report.probeHeight !== undefined || report.normalizedHeight !== undefined) { console.log( ` probe heights: probe ${Math.round(report.probeHeight ?? 0)}px | normalized ${Math.round(report.normalizedHeight ?? 0)}px | book ${actual}px | method ${report.browserLineMethod ?? '-'}`, ) } if (report.extractorSensitivity !== null && report.extractorSensitivity !== undefined) { console.log(` extractor sensitivity: ${report.extractorSensitivity}`) } if ( report.alternateBrowserLineMethod !== undefined && report.alternateBrowserLineCount !== undefined ) { console.log( ` alternate method: ${report.alternateBrowserLineMethod} (${report.predictedLineCount ?? '-'}${report.alternateBrowserLineCount !== undefined ? `/${report.alternateBrowserLineCount}` : ''} lines)` + (report.alternateFirstBreakMismatch === null ? ' exact' : ''), ) } if (report.maxDriftLine !== null && report.maxDriftLine !== undefined) { console.log( ` drift sample L${report.maxDriftLine.line}: ${report.maxDriftLine.drift.toFixed(3)}px (${report.maxDriftLine.sumWidth.toFixed(3)} sum vs ${report.maxDriftLine.fullWidth.toFixed(3)} full vs ${report.maxDriftLine.domWidth.toFixed(3)} dom vs ${report.maxDriftLine.pairAdjustedWidth.toFixed(3)} pair)`, ) console.log(` text: ${JSON.stringify(report.maxDriftLine.text.slice(0, 140))}`) if (report.maxDriftLine.segments.length > 0) { const summary = report.maxDriftLine.segments .map(segment => `${JSON.stringify(segment.text)}@${segment.width.toFixed(2)}/${segment.domWidth.toFixed(2)}${segment.isSpace ? ':space' : ''}`) .join(' | ') console.log(` segments: ${summary}`) } } if (report.firstBreakMismatch !== null && report.firstBreakMismatch !== undefined) { const mismatch = report.firstBreakMismatch console.log(` break L${mismatch.line}: ${mismatch.reasonGuess}`) console.log(` offsets: ours ${mismatch.oursStart}-${mismatch.oursEnd} | browser ${mismatch.browserStart}-${mismatch.browserEnd}`) console.log(` delta: ${JSON.stringify(mismatch.deltaText)}`) console.log(` ours text: ${JSON.stringify(mismatch.oursText)}`) console.log(` browser text: ${JSON.stringify(mismatch.browserText)}`) console.log(` ours rendered: ${JSON.stringify(mismatch.oursRenderedText)}`) console.log(` browser rendered: ${JSON.stringify(mismatch.browserRenderedText)}`) ad:T1eaf,Command: /bin/zsh -lc "sed -n '1,260p' pages/probe.ts" Chunk ID: c73c92 Wall time: 0.0000 seconds Process exited with code 0 Original token count: 1924 Output: import { layout, layoutWithLines, prepareWithSegments, type PreparedTextWithSegments, } from '../src/layout.ts' import { formatBreakContext, getDiagnosticUnits, getLineContent, measureCanvasTextWidth, measureDomTextWidth, } from './diagnostic-utils.ts' import { clearNavigationReport, publishNavigationPhase, publishNavigationReport } from './report-utils.ts' type ProbeLine = { text: string renderedText: string contentText: string start: number end: number contentEnd: number fullWidth: number domWidth: number sumWidth?: number } type ProbeBreakMismatch = { line: number oursStart: number browserStart: number oursEnd: number browserEnd: number oursText: string browserText: string oursRenderedText: string browserRenderedText: string oursContext: string browserContext: string deltaText: string reasonGuess: string oursSumWidth: number oursDomWidth: number oursFullWidth: number browserDomWidth: number browserFullWidth: number } type ProbeLineSummary = { line: number text: string renderedText: string start: number end: number } type ProbeReport = { status: 'ready' | 'error' requestId?: string text?: string whiteSpace?: 'normal' | 'pre-wrap' width?: number contentWidth?: number font?: string lineHeight?: number direction?: string browserLineMethod?: 'range' | 'span' predictedHeight?: number actualHeight?: number diffPx?: number predictedLineCount?: number browserLineCount?: number firstBreakMismatch?: ProbeBreakMismatch | null alternateBrowserLineMethod?: 'range' | 'span' alternateBrowserLineCount?: number alternateFirstBreakMismatch?: ProbeBreakMismatch | null extractorSensitivity?: string | null ourLines?: ProbeLineSummary[] browserLines?: ProbeLineSummary[] alternateBrowserLines?: ProbeLineSummary[] message?: string } declare global { interface Window { __PROBE_REPORT__?: ProbeReport } } const PADDING = 40 const params = new URLSearchParams(location.search) const requestId = params.get('requestId') ?? undefined const text = params.get('text') ?? '' const width = Math.max(100, Number.parseInt(params.get('width') ?? '600', 10)) const font = params.get('font') ?? '18px serif' const lineHeight = Math.max(1, Number.parseInt(params.get('lineHeight') ?? '32', 10)) const direction = params.get('dir') === 'rtl' ? 'rtl' : 'ltr' const lang = params.get('lang') ?? (direction === 'rtl' ? 'ar' : 'en') const browserLineMethod = params.get('method') === 'span' ? 'span' : 'range' const verbose = params.get('verbose') === '1' const whiteSpace = params.get('whiteSpace') === 'pre-wrap' ? 'pre-wrap' : 'normal' const cssWhiteSpace = whiteSpace === 'pre-wrap' ? 'pre-wrap' : 'normal' const stats = document.getElementById('stats')! const book = document.getElementById('book')! const diagnosticDiv = document.createElement('div') diagnosticDiv.style.position = 'absolute' diagnosticDiv.style.top = '-99999px' diagnosticDiv.style.left = '-99999px' diagnosticDiv.style.visibility = 'hidden' diagnosticDiv.style.pointerEvents = 'none' diagnosticDiv.style.boxSizing = 'border-box' diagnosticDiv.style.whiteSpace = cssWhiteSpace diagnosticDiv.style.wordWrap = 'break-word' diagnosticDiv.style.overflowWrap = 'break-word' diagnosticDiv.style.padding = `${PADDING}px` document.body.appendChild(diagnosticDiv) const diagnosticCanvas = document.createElement('canvas') const diagnosticCtx = diagnosticCanvas.getContext('2d')! const graphemeSegmenter = new Intl.Segmenter(undefined, { granularity: 'grapheme' }) function withRequestId(report: T): ProbeReport { return requestId === undefined ? report : { ...report, requestId } } function publishReport(report: ProbeReport): void { window.__PROBE_REPORT__ = report publishNavigationReport(report) } function setError(message: string): void { stats.textContent = `Error: ${message}` publishReport(withRequestId({ status: 'error', message })) } function getBrowserLinesFromSpans(prepared: PreparedTextWithSegments, measuredFont: string, dir: string): ProbeLine[] { const lines: ProbeLine[] = [] const units = getDiagnosticUnits(prepared) const spans: HTMLSpanElement[] = [] let currentLine = '' let currentStart: number | null = null let currentEnd = 0 let lastTop: number | null = null diagnosticDiv.textContent = '' for (const unit of units) { const span = document.createElement('span') span.textContent = unit.text diagnosticDiv.appendChild(span) spans.push(span) } function pushLine(): void { if (currentStart === null || currentLine.length === 0) return const content = getLineContent(currentLine, currentEnd) lines.push({ text: currentLine, renderedText: currentLine, contentText: content.text, start: currentStart, end: currentEnd, contentEnd: content.end, fullWidth: measureCanvasTextWidth(diagnosticCtx, content.text, measuredFont), domWidth: measureDomTextWidth(document, content.text, measuredFont, dir), }) } for (let i = 0; i < units.length; i++) { const unit = units[i]! const span = spans[i]! const rect = span.getBoundingClientRect() const top: number | null = rect.width > 0 || rect.height > 0 ? rect.top : lastTop if (top !== null && lastTop !== null && top > lastTop + 0.5) { pushLine() currentLine = unit.text currentStart = unit.start currentEnd = unit.end } else { if (currentStart === null) currentStart = unit.start currentLine += unit.text currentEnd = unit.end } if (top !== null) lastTop = top } pushLine() diagnosticDiv.textContent = text return lines } function getBrowserLines( prepared: PreparedTextWithSegments, measuredFont: string, dir: string, method: 'range' | 'span', ): ProbeLine[] { return method === 'span' ? getBrowserLinesFromSpans(prepared, measuredFont, dir) : getBrowserLinesFromRange(prepared, measuredFont, dir) } function getBrowserLinesFromRange(prepared: PreparedTextWithSegments, measuredFont: string, dir: string): ProbeLine[] { const textNode = diagnosticDiv.firstChild const lines: ProbeLine[] = [] if (!(textNode instanceof Text)) return lines const units = getDiagnosticUnits(prepared) const range = document.createRange() let currentLine = '' let currentStart: number | null = null let currentEnd = 0 let lastTop: number | null = null function pushLine(): void { if (currentStart === null || currentLine.length === 0) return const content = getLineContent(currentLine, currentEnd) lines.push({ text: currentLine, renderedText: currentLine, contentText: content.text, start: currentStart, end: currentEnd, contentEnd: content.end, fullWidth: measureCanvasTextWidth(diagnosticCtx, content.text, measuredFont), domWidth: measureDomTextWidth(document, content.text, measuredFont, dir), }) } for (const unit of units) { range.setStart(textNode, unit.start) range.setEnd(textNode, unit.end) const rects = range.getClientRects() const top: number | null = rects.length > 0 ? rects[0]!.top : lastTop if (top !== null && lastTop !== null && top > lastTop + 0.5) { pushLine() currentLine = unit.text currentStart = unit.start currentEnd = unit.end } else { if (currentStart === null) currentStart = unit.start currentLine += unit.text currentEnd = unit.end } if (top !== null) lastTop = top } pushLine() return lines } function measurePreparedSlice( prepared: PreparedTextWithSegments, start: number, end: number, ae:Tbce6,Command: /bin/zsh -lc 'rg -n "prepare|layoutWithLines|layoutNextLine|walkLineRanges|prepareWithSegments|walkPreparedLines|wrapGeometry|cursor|virtual|masonry|balance|justify" pages/demos pages/*.ts' Chunk ID: 595917 Wall time: 0.0001 seconds Process exited with code 0 Original token count: 69674 Output: Total output lines: 311 pages/diagnostic-utils.ts:11:export function getDiagnosticUnits(prepared: PreparedTextWithSegments): DiagnosticUnit[] { pages/diagnostic-utils.ts:15: for (let i = 0; i < prepared.segments.length; i++) { pages/diagnostic-utils.ts:16: const segment = prepared.segments[i]! pages/diagnostic-utils.ts:17: if (prepared.breakableWidths[i] !== null) { pages/gatsby.ts:3: layoutWithLines, pages/gatsby.ts:4: prepareWithSegments, pages/gatsby.ts:209:const prepared = prepareWithSegments(text, FONT) pages/gatsby.ts:210:const segmentSpans = buildSegmentSpans(prepared) pages/gatsby.ts:303:function buildSegmentSpans(preparedText: PreparedTextWithSegments): SegmentSpan[] { pages/gatsby.ts:307: for (let i = 0; i < preparedText.segments.length; i++) { pages/gatsby.ts:308: const segText = preparedText.segments[i]! pages/gatsby.ts:316: width: preparedText.widths[i]!, pages/gatsby.ts:317: isSpace: preparedText.kinds[i] === 'space', pages/gatsby.ts:318: breakable: preparedText.breakableWidths[i] !== null, pages/gatsby.ts:424: preparedText: PreparedTextWithSegments, pages/gatsby.ts:431: const units = getDiagnosticUnits(preparedText) pages/gatsby.ts:485: preparedText: PreparedTextWithSegments, pages/gatsby.ts:488: const layoutLines = layoutWithLines(preparedText, maxWidth, LINE_HEIGHT).lines pages/gatsby.ts:615: const ourLines = getOurLines(prepared, contentWidth) pages/gatsby.ts:616: const browserLines = getBrowserLines(prepared, diagnosticDiv) pages/gatsby.ts:659: const predicted = layout(prepared, contentWidth, LINE_HEIGHT) pages/benchmark.ts:2: prepare, pages/benchmark.ts:3: prepareWithSegments, pages/benchmark.ts:5: layoutNextLine, pages/benchmark.ts:6: layoutWithLines, pages/benchmark.ts:7: walkLineRanges, pages/benchmark.ts:67: prepareMs: number pages/benchmark.ts:263: const prepareSamples: number[] = [] pages/benchmark.ts:271: prepareSamples.push(profile.totalMs) pages/benchmark.ts:277: const prepareMs = median(prepareSamples) pages/benchmark.ts:282: const prepared = prepareWithSegments(corpus.text, corpus.font) pages/benchmark.ts:283: const lineCount = layout(prepared, corpus.width, corpus.lineHeight).lineCount pages/benchmark.ts:287: const result = layout(prepared, width, corpus.lineHeight) pages/benchmark.ts:297: segments: prepared.widths.length, pages/benchmark.ts:298: breakableSegments: prepared.breakableWidths.filter(widths => widths !== null).length, pages/benchmark.ts:303: prepareMs, pages/benchmark.ts:313: prepared: PreparedTextWithSegments[], pages/benchmark.ts:321: const layoutWithLinesMs = bench(repeatIndex => { pages/benchmark.ts:324: for (let i = 0; i < prepared.length; i++) { pages/benchmark.ts:325: const result = layoutWithLines(prepared[i]!, width, lineHeight) pages/benchmark.ts:331: const walkLineRangesMs = bench(repeatIndex => { pages/benchmark.ts:334: for (let i = 0; i < prepared.length; i++) { pages/benchmark.ts:335: sum += walkLineRanges(prepared[i]!, width, line => { pages/benchmark.ts:342: const layoutNextLineMs = bench(repeatIndex => { pages/benchmark.ts:345: for (let i = 0; i < prepared.length; i++) { pages/benchmark.ts:346: let cursor = { segmentIndex: 0, graphemeIndex: 0 } pages/benchmark.ts:348: const line = layoutNextLine(prepared[i]!, cursor, width) pages/benchmark.ts:351: cursor = line.end pages/benchmark.ts:361: label: 'Our library: layoutWithLines()', pages/benchmark.ts:362: ms: layoutWithLinesMs, pages/benchmark.ts:366: label: 'Our library: walkLineRanges()', pages/benchmark.ts:367: ms: walkLineRangesMs, pages/benchmark.ts:371: label: 'Our library: layoutNextLine()', pages/benchmark.ts:372: ms: layoutNextLineMs, pages/benchmark.ts:431: // Pre-prepare for layout benchmark pages/benchmark.ts:432: const prepared: PreparedText[] = [] pages/benchmark.ts:434: prepared.push(prepare(texts[i]!, FONT)) pages/benchmark.ts:441: // --- 1. prepare() --- pages/benchmark.ts:442: root.innerHTML = '

Benchmarking prepare()...

' pages/benchmark.ts:447: prepare(texts[i]!, FONT) pages/benchmark.ts:450: results.push({ label: 'Our library: prepare()', ms: tPrepare, desc: `One cold ${COUNT}-text measurement batch` }) pages/benchmark.ts:459: const result = layout(prepared[i]!, maxWidth, LINE_HEIGHT) pages/benchmark.ts:504: const richPrepared = richTexts.map(text => prepareWithSegments(text, FONT)) pages/benchmark.ts:518: prepareWithSegments( pages/benchmark.ts:537: // Relative speed only for resize approaches (layout vs DOM). prepare() is pages/benchmark.ts:556:

${RICH_COUNT} shared-corpus texts prepared with segments. Median ms per batch across widths ${RICH_LAYOUT_SAMPLE_WIDTHS.join('/')}px. This tracks the richer APIs used by shrinkwrap, custom layout, and manual reflow.

pages/benchmark.ts:561:

${RICH_LONG_REPEAT} copies of the Arabic long-form corpus prepared with segments. Median ms per batch across widths ${RICH_LONG_SAMPLE_WIDTHS.join('/')}px. This is the richer-path worst-case canary.

pages/benchmark.ts:586: // prepare (cold) pages/benchmark.ts:589: clearCache(); let t0 = performance.now(); prepare(cjk, FONT); pTimes.cjk.push(performance.now() - t0) pages/benchmark.ts:590: clearCache(); t0 = performance.now(); prepare(lat, FONT); pTimes.lat.push(performance.now() - t0) pages/benchmark.ts:595: const pc = prepareWithSegments(cjk, FONT) pages/benchmark.ts:596: const pl = prepareWithSegments(lat, FONT) pages/benchmark.ts:629: CharsCJK segsLatin segsCJK prepare (ms)Latin prepare (ms)CJK layout/1k (ms)Latin layout/1k (ms) pages/benchmark.ts:644: ${result.prepareMs.toFixed(2)} pages/benchmark.ts:650:

Long-form rows split cold prepare into text analysis and measurement phases for one full corpus text, then report one hot layout pass over the prepared result. They are intended to catch script-specific prepare regressions that the short shared corpus can hide.