告别 Pagefind，用 Orama 实现静态博客的全文搜索

[features]
extended = ["dep:charabia"]
[dependencies]
charabia = { version = "0.9.3", optional = true, default-features = false, features = ["chinese","japanese","thai"] }

const segmenter = new Intl.Segmenter("zh-CN", { granularity: "word" });
function tokenize(text: string): string[] {
    const segments = segmenter.segment(text);
    const tokens: string[] = [];
    for (const segment of segments) {
        if (segment.isWordLike) {
            tokens.push(segment.segment);
        }
    }
    return tokens;
}

export function BM25(
    tf: number, // 词在文档中的频率
    matchingCount: number, // 包含该词的文档数
    docsCount: number, // 总文档数
    fieldLength: number, // 该文档字段长度
    averageFieldLength: number, // 平均字段长度
    { k, b, d }: Required<BM25Params>,
) {
    const idf = Math.log(1 + (docsCount - matchingCount + 0.5) / (matchingCount + 0.5));
    return (idf * (d + tf * (k + 1))) / (tf + k * (1 - b + (b * fieldLength) / averageFieldLength));
}

ENOENT: no such file or directory, mkdir '/.../%E6%A1%8C%E9%9D%A2/.../dist/assets'

import { create, insertMultiple, save } from "@orama/orama";
import { createTokenizer } from "@orama/tokenizers/mandarin";
import { stopwords as mandarinStopwords } from "@orama/stopwords/mandarin";
import { getCollection } from "astro:content";
import removeMarkdown from "remove-markdown";
const schema = {
    id: "string",
    type: "string",
    title: "string",
    description: "string",
    content: "string",
    url: "string",
    date: "string",
    tags: "string[]",
} as const;
export async function GET() {
    const [posts, aboutPages, wordEntries] = await Promise.all([
        getCollection("posts", ({ data }) => !data.draft),
        getCollection("about"),
        getCollection("words", ({ data }) => !data.draft),
    ]);
    const documents = [];
    for (const post of posts) {
        const body = typeof post.body === "string" ? post.body : "";
        documents.push({
            id: `post-${post.data.abbrlink}`,
            type: "post",
            title: post.data.title,
            description: post.data.desc,
            content: removeMarkdown(body), // 清理 markdown 语法
            url: `/posts/${post.data.abbrlink}/`,
            date: post.data.date ?? "",
            tags: post.data.tags ?? [],
        });
    }
    // ... about 和 words 类似处理
    const db = create({
        schema,
        components: {
            tokenizer: createTokenizer({
                // 中文分词
                stopWords: mandarinStopwords, // 中文停用词
            }),
        },
    });
    insertMultiple(db, documents);
    const index = save(db);
    return new Response(JSON.stringify(index), {
        headers: { "Content-Type": "application/json" },
    });
}

<script>
    import { create, load, search } from "@orama/orama";
    import { createTokenizer } from "@orama/tokenizers/mandarin";
    import { stopwords as mandarinStopwords } from "@orama/stopwords/mandarin";
    import { Highlight, highlightStrategy } from "@orama/highlight";
    const schema = {
        id: "string", type: "string", title: "string",
        description: "string", content: "string",
        url: "string", date: "string", tags: "string[]",
    } as const;
    async function initOrama() {
        const response = await fetch("/search-index.json");
        const data = await response.json();
        const instance = create({
            schema,
            components: {
                tokenizer: createTokenizer({
                    stopWords: mandarinStopwords,
                }),
            },
        });
        load(instance, data);
        return instance;
    }
    async function performSearch(query: string) {
        const db = await initOrama();
        const results = await search(db, {
            term: query,
            limit: 20,
            threshold: 0,    // 只返回包含所有查询词的文档
            properties: ["title", "description", "content", "tags"],
            boost: { title: 2, tags: 1.5 },
        });
        renderResults(results.hits, query);
    }
    function renderResults(hits, query) {
        for (const hit of hits) {
            const dHL = new Highlight({
                HTMLTag: "mark",
                strategy: highlightStrategy.PARTIAL_MATCH,
            });
            dHL.highlight(hit.document.content ?? hit.document.description, query);
            // trim(200) 自动居中到第一个匹配位置
            const snippet = dHL.positions.length > 0
                ? dHL.trim(200)
                : (dHL.HTML ?? "").substring(0, 200);
            // 设置 innerHTML 显示高亮
        }
    }
</script>

"astro-pagefind": "^2.0.0"
"@orama/orama": "^3.1.18"
"@orama/tokenizers": "^3.1.18"
"@orama/stopwords": "^3.1.18"
"@orama/highlight": "^0.1.9"
"remove-markdown": "^0.6.4"