| 1 | // Reading the current page as text, so it can be handed to an agent in the pty. |
| 2 | // |
| 3 | // Same contract as picker.js: this function is serialised and executed in the |
| 4 | // page's world via scripting.executeScript, so it must be entirely |
| 5 | // self-contained — no imports, no closures over extension state, no extension |
| 6 | // privileges. What it returns comes back as executeScript's *return value*, |
| 7 | // which is why the sidebar needs no inbound message listener a content script |
| 8 | // could reach. |
| 9 | // |
| 10 | // Everything it returns is page-controlled data. It is not text until |
| 11 | // Sanitize.forPasteBlock has been over it — see sidebar.js. |
| 12 | |
| 13 | /** |
| 14 | * @param {number} max hard cap on how many characters of body text to return |
| 15 | */ |
| 16 | function tbReadPage(max) { |
| 17 | // Elements that are never prose. SVG and CANVAS carry no readable text; |
| 18 | // SCRIPT and STYLE carry text that is not prose; the rest are chrome. |
| 19 | const SKIP = new Set([ |
| 20 | "SCRIPT", |
| 21 | "STYLE", |
| 22 | "NOSCRIPT", |
| 23 | "TEMPLATE", |
| 24 | "IFRAME", |
| 25 | "SVG", |
| 26 | "CANVAS", |
| 27 | "AUDIO", |
| 28 | "VIDEO", |
| 29 | "SELECT", |
| 30 | "OPTION", |
| 31 | "TEXTAREA", |
| 32 | ]); |
| 33 | |
| 34 | // Page furniture: dropped only when we are reading the whole document, since |
| 35 | // then they are the site's navigation rather than the article. Inside a |
| 36 | // <main> or <article> they are the author's own structure and stay. |
| 37 | const FURNITURE = new Set(["NAV", "HEADER", "FOOTER", "ASIDE"]); |
| 38 | |
| 39 | // Anything that ends a line of prose. Used two ways: to know which elements |
| 40 | // produce a block, and — as a selector — to know whether an element is a |
| 41 | // container to recurse into or a leaf to read. |
| 42 | const BLOCK = |
| 43 | "p,h1,h2,h3,h4,h5,h6,li,pre,blockquote,figcaption,dt,dd,td,th,tr,table,ul,ol,article,section,main,div"; |
| 44 | |
| 45 | /** @param {string} s */ |
| 46 | const squash = (s) => s.replace(/\s+/g, " ").trim(); |
| 47 | |
| 48 | /** |
| 49 | * Rendered, in the sense that matters here: it has boxes. Catches |
| 50 | * `display: none`, `hidden`, and the zero-size wrappers a framework leaves |
| 51 | * behind, without a getComputedStyle call per element. |
| 52 | * |
| 53 | * @param {Element} el |
| 54 | */ |
| 55 | function rendered(el) { |
| 56 | if (el.getClientRects().length) return true; |
| 57 | // A <tr> in a table with `display: block` cells, and other layout oddities, |
| 58 | // can report no rects while its contents are plainly visible. Fall back to |
| 59 | // the cheap style read rather than dropping real text. |
| 60 | const cs = el.ownerDocument.defaultView?.getComputedStyle(el); |
| 61 | return !cs || (cs.display !== "none" && cs.visibility !== "hidden"); |
| 62 | } |
| 63 | |
| 64 | /** @type {string[]} */ |
| 65 | const lines = []; |
| 66 | let chars = 0; |
| 67 | let full = true; |
| 68 | |
| 69 | /** @param {string} line */ |
| 70 | function emit(line) { |
| 71 | const text = line.trimEnd(); |
| 72 | if (!text) return; |
| 73 | // Site furniture repeats itself — a heading that is also the link that got |
| 74 | // you here, a label rendered twice at two breakpoints. Consecutive |
| 75 | // duplicates are always noise. |
| 76 | if (lines[lines.length - 1] === text) return; |
| 77 | if (chars + text.length > max) { |
| 78 | full = false; |
| 79 | return; |
| 80 | } |
| 81 | lines.push(text); |
| 82 | chars += text.length + 1; |
| 83 | } |
| 84 | |
| 85 | /** |
| 86 | * Text of an element, minus anything that would be read as its own block. |
| 87 | * A list item's own words without its nested list, a cell's without a table |
| 88 | * inside it. |
| 89 | * |
| 90 | * @param {Element} el |
| 91 | */ |
| 92 | function ownText(el) { |
| 93 | let out = ""; |
| 94 | for (const node of el.childNodes) { |
| 95 | if (node.nodeType === 3) out += node.nodeValue ?? ""; |
| 96 | else if (node.nodeType === 1) { |
| 97 | const child = /** @type {Element} */ (node); |
| 98 | if (SKIP.has(child.tagName)) continue; |
| 99 | if (child.matches("ul,ol,table,pre,p,li,h1,h2,h3,h4,h5,h6,blockquote")) continue; |
| 100 | out += ownText(child); |
| 101 | } |
| 102 | } |
| 103 | return squash(out); |
| 104 | } |
| 105 | |
| 106 | /** |
| 107 | * @param {Element} el |
| 108 | * @param {boolean} trimFurniture whether NAV/HEADER/FOOTER/ASIDE are chrome |
| 109 | */ |
| 110 | function walk(el, trimFurniture) { |
| 111 | if (!full) return; |
| 112 | const tag = el.tagName; |
| 113 | if (SKIP.has(tag)) return; |
| 114 | if (trimFurniture && FURNITURE.has(tag)) return; |
| 115 | if (el.getAttribute("aria-hidden") === "true") return; |
| 116 | if (!rendered(el)) return; |
| 117 | |
| 118 | if (/^H[1-6]$/.test(tag)) { |
| 119 | emit(`${"#".repeat(Number(tag[1]))} ${squash(el.textContent ?? "")}`); |
| 120 | return; |
| 121 | } |
| 122 | if (tag === "PRE") { |
| 123 | const code = (el.textContent ?? "").replace(/\s+$/, ""); |
| 124 | if (code) emit(`\`\`\`\n${code}\n\`\`\``); |
| 125 | return; |
| 126 | } |
| 127 | if (tag === "LI") { |
| 128 | emit(`- ${ownText(el)}`); |
| 129 | // The nested list is its own set of items, so it is walked rather than |
| 130 | // folded into the line above. |
| 131 | for (const child of el.children) { |
| 132 | if (child.matches("ul,ol,table,pre")) walk(child, trimFurniture); |
| 133 | } |
| 134 | return; |
| 135 | } |
| 136 | if (tag === "TR") { |
| 137 | /** @type {string[]} */ |
| 138 | const cells = []; |
| 139 | for (const cell of el.children) cells.push(ownText(cell)); |
| 140 | if (cells.some(Boolean)) emit(cells.join(" | ")); |
| 141 | return; |
| 142 | } |
| 143 | if (tag === "P" || tag === "BLOCKQUOTE" || tag === "FIGCAPTION" || tag === "DT" || tag === "DD") { |
| 144 | emit(ownText(el)); |
| 145 | return; |
| 146 | } |
| 147 | |
| 148 | // Not a block of its own: either a container of blocks, which is walked, or |
| 149 | // a leaf that happens to hold the text — a <div> or <span> with words in it |
| 150 | // and no <p> in sight, which is most of the modern web. |
| 151 | if (el.querySelector(BLOCK)) { |
| 152 | for (const child of el.children) walk(child, trimFurniture); |
| 153 | // Text sitting directly inside a container, alongside its blocks, would |
| 154 | // otherwise be dropped. |
| 155 | const own = ownText(el); |
| 156 | if (own) emit(own); |
| 157 | return; |
| 158 | } |
| 159 | emit(squash(el.textContent ?? "")); |
| 160 | } |
| 161 | |
| 162 | /** @param {string} sel */ |
| 163 | const meta = (sel) => squash(document.querySelector(sel)?.getAttribute("content") ?? ""); |
| 164 | |
| 165 | const selection = squash(window.getSelection()?.toString() ?? ""); |
| 166 | |
| 167 | if (selection) { |
| 168 | // A selection is the user saying which part they meant, so it is taken |
| 169 | // verbatim — no extraction, no furniture trimming, no reordering. |
| 170 | const truncated = selection.length > max; |
| 171 | return { |
| 172 | url: location.href, |
| 173 | title: document.title || "", |
| 174 | description: meta('meta[name="description"]') || meta('meta[property="og:description"]'), |
| 175 | kind: /** @type {"selection"} */ ("selection"), |
| 176 | body: truncated ? selection.slice(0, max) : selection, |
| 177 | chars: selection.length, |
| 178 | truncated, |
| 179 | }; |
| 180 | } |
| 181 | |
| 182 | // The narrowest thing that plausibly holds the article. Falling back to |
| 183 | // <body> is the common case; the trim of nav/header/footer only applies then, |
| 184 | // because inside a <main> those tags are the author's own. |
| 185 | const main = document.querySelector("main, article, [role='main']"); |
| 186 | const root = main ?? document.body ?? document.documentElement; |
| 187 | walk(root, !main); |
| 188 | |
| 189 | return { |
| 190 | url: location.href, |
| 191 | title: document.title || "", |
| 192 | description: meta('meta[name="description"]') || meta('meta[property="og:description"]'), |
| 193 | kind: /** @type {"page"} */ ("page"), |
| 194 | body: lines.join("\n"), |
| 195 | chars, |
| 196 | truncated: !full, |
| 197 | }; |
| 198 | } |