reya/lume
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move the event parser and dedup functions to Rust (#206)

Browse Source 
* feat: improve js parser

* feat: move parser and dedup to rust

* fix: parser

* fix: get event function

* feat: improve parser performance (#207)

* feat: improve parser performance

* feat: add test for video parsing

* feat: finish new parser

---------

Co-authored-by: XIAO YU <xyzmhx@gmail.com>
This commit is contained in:
雨宮蓮
2024-06-12 08:27:53 +07:00
committed by GitHub
parent 1c20512ecc
commit 71be59b2e9
21 changed files with 610 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
packages/system/src/commands.ts
View File

@@ -244,7 +244,7 @@ try {
else return { status: "error", error: e as any };
}
},
async getEvent(id: string) : Promise<Result<string, string>> {
async getEvent(id: string) : Promise<Result<RichEvent, string>> {
try {
return { status: "ok", data: await TAURI_INVOKE("get_event", { id }) };
} catch (e) {
@@ -252,7 +252,7 @@ try {
else return { status: "error", error: e as any };
}
},
async getReplies(id: string) : Promise<Result<string[], string>> {
async getReplies(id: string) : Promise<Result<RichEvent[], string>> {
try {
return { status: "ok", data: await TAURI_INVOKE("get_replies", { id }) };
} catch (e) {
@@ -260,7 +260,7 @@ try {
else return { status: "error", error: e as any };
}
},
async getEventsBy(publicKey: string, asOf: string | null) : Promise<Result<string[], string>> {
async getEventsBy(publicKey: string, asOf: string | null) : Promise<Result<RichEvent[], string>> {
try {
return { status: "ok", data: await TAURI_INVOKE("get_events_by", { publicKey, asOf }) };
} catch (e) {
@@ -268,7 +268,7 @@ try {
else return { status: "error", error: e as any };
}
},
async getLocalEvents(pubkeys: string[], until: string | null) : Promise<Result<string[], string>> {
async getLocalEvents(pubkeys: string[], until: string | null) : Promise<Result<RichEvent[], string>> {
try {
return { status: "ok", data: await TAURI_INVOKE("get_local_events", { pubkeys, until }) };
} catch (e) {
@@ -276,7 +276,7 @@ try {
else return { status: "error", error: e as any };
}
},
async getGlobalEvents(until: string | null) : Promise<Result<string[], string>> {
async getGlobalEvents(until: string | null) : Promise<Result<RichEvent[], string>> {
try {
return { status: "ok", data: await TAURI_INVOKE("get_global_events", { until }) };
} catch (e) {
@@ -284,7 +284,7 @@ try {
else return { status: "error", error: e as any };
}
},
async getHashtagEvents(hashtags: string[], until: string | null) : Promise<Result<string[], string>> {
async getHashtagEvents(hashtags: string[], until: string | null) : Promise<Result<RichEvent[], string>> {
try {
return { status: "ok", data: await TAURI_INVOKE("get_hashtag_events", { hashtags, until }) };
} catch (e) {
@@ -367,7 +367,9 @@ await TAURI_INVOKE("set_badge", { count });
/** user-defined types **/
export type Account = { npub: string; nsec: string }
export type Meta = { content: string; images: string[]; videos: string[]; events: string[]; mentions: string[]; hashtags: string[] }
export type Relays = { connected: string[]; read: string[] | null; write: string[] | null; both: string[] | null }
export type RichEvent = { raw: string; parsed: Meta | null }
/** tauri-specta globals **/

19
packages/system/src/event.ts
View File

@@ -1,4 +1,4 @@
import { EventWithReplies, Kind, NostrEvent } from "@lume/types";
import type { EventWithReplies, Kind, Meta, NostrEvent } from "@lume/types";
import { commands } from "./commands";
import { generateContentTags } from "@lume/utils";
@@ -11,6 +11,7 @@ export class LumeEvent {
public content: string;
public sig: string;
public relay?: string;
public meta: Meta;
#raw: NostrEvent;
constructor(event: NostrEvent) {
@@ -74,9 +75,17 @@ export class LumeEvent {
const query = await commands.getReplies(id);
if (query.status === "ok") {
const events = query.data.map(
(item) => JSON.parse(item) as EventWithReplies,
);
const events = query.data.map((item) => {
const raw = JSON.parse(item.raw) as EventWithReplies;
if (item.parsed) {
raw.meta = item.parsed;
} else {
raw.meta = null;
}
return raw;
});
if (events.length > 0) {
const replies = new Set();
@@ -135,7 +144,7 @@ export class LumeEvent {
const queryReply = await commands.getEvent(reply_to);
if (queryReply.status === "ok") {
const replyEvent = JSON.parse(queryReply.data) as NostrEvent;
const replyEvent = JSON.parse(queryReply.data.raw) as NostrEvent;
const relayHint =
replyEvent.tags.find((ev) => ev[0] === "e")?.[0][2] ?? "";

10
packages/system/src/hooks/useEvent.ts
View File

@@ -1,18 +1,12 @@
import type { Event, NostrEvent } from "@lume/types";
import { useQuery } from "@tanstack/react-query";
import { invoke } from "@tauri-apps/api/core";
import { NostrQuery } from "../query";
export function useEvent(id: string) {
const { isLoading, isError, data } = useQuery({
queryKey: ["event", id],
queryFn: async () => {
try {
const eventId: string = id
.replace("nostr:", "")
.split("'")[0]
.split(".")[0];
const cmd: string = await invoke("get_event", { id: eventId });
const event: NostrEvent = JSON.parse(cmd);
const event = await NostrQuery.getEvent(id);
return event;
} catch (e) {
throw new Error(e);

83
packages/system/src/query.ts
View File

@@ -1,10 +1,15 @@
import { LumeColumn, Metadata, NostrEvent, Relay, Settings } from "@lume/types";
import type {
LumeColumn,
Metadata,
NostrEvent,
Relay,
Settings,
} from "@lume/types";
import { commands } from "./commands";
import { resolveResource } from "@tauri-apps/api/path";
import { readFile, readTextFile } from "@tauri-apps/plugin-fs";
import { isPermissionGranted } from "@tauri-apps/plugin-notification";
import { open } from "@tauri-apps/plugin-dialog";
import { dedupEvents } from "./dedup";
import { invoke } from "@tauri-apps/api/core";
import { relaunch } from "@tauri-apps/plugin-process";
@@ -98,9 +103,16 @@ export class NostrQuery {
const query = await commands.getEvent(normalize);
if (query.status === "ok") {
const event: NostrEvent = JSON.parse(query.data);
return event;
const data = query.data;
const raw = JSON.parse(data.raw) as NostrEvent;
if (data?.parsed) {
raw.meta = data.parsed;
}
return raw;
} else {
console.log("[getEvent]: ", query.error);
return null;
}
}
@@ -110,8 +122,19 @@ export class NostrQuery {
const query = await commands.getEventsBy(pubkey, until);
if (query.status === "ok") {
const events = query.data.map((item) => JSON.parse(item) as NostrEvent);
return events;
const data = query.data.map((item) => {
const raw = JSON.parse(item.raw) as NostrEvent;
if (item.parsed) {
raw.meta = item.parsed;
} else {
raw.meta = null;
}
return raw;
});
return data;
} else {
return [];
}
@@ -122,10 +145,19 @@ export class NostrQuery {
const query = await commands.getLocalEvents(pubkeys, until);
if (query.status === "ok") {
const events = query.data.map((item) => JSON.parse(item) as NostrEvent);
const dedup = dedupEvents(events);
const data = query.data.map((item) => {
const raw = JSON.parse(item.raw) as NostrEvent;
return dedup;
if (item.parsed) {
raw.meta = item.parsed;
} else {
raw.meta = null;
}
return raw;
});
return data;
} else {
return [];
}
@@ -136,10 +168,19 @@ export class NostrQuery {
const query = await commands.getGlobalEvents(until);
if (query.status === "ok") {
const events = query.data.map((item) => JSON.parse(item) as NostrEvent);
const dedup = dedupEvents(events);
const data = query.data.map((item) => {
const raw = JSON.parse(item.raw) as NostrEvent;
return dedup;
if (item.parsed) {
raw.meta = item.parsed;
} else {
raw.meta = null;
}
return raw;
});
return data;
} else {
return [];
}
@@ -151,10 +192,19 @@ export class NostrQuery {
const query = await commands.getHashtagEvents(nostrTags, until);
if (query.status === "ok") {
const events = query.data.map((item) => JSON.parse(item) as NostrEvent);
const dedup = dedupEvents(events);
const data = query.data.map((item) => {
const raw = JSON.parse(item.raw) as NostrEvent;
return dedup;
if (item.parsed) {
raw.meta = item.parsed;
} else {
raw.meta = null;
}
return raw;
});
return data;
} else {
return [];
}
@@ -311,8 +361,7 @@ export class NostrQuery {
const query = await commands.getBootstrapRelays();
if (query.status === "ok") {
let relays: Relay[] = [];
console.log(query.data);
const relays: Relay[] = [];
for (const item of query.data) {
const line = item.split(",");

10
packages/types/index.d.ts vendored
View File

@@ -28,6 +28,15 @@ export enum Kind {
// #TODO: Add all nostr kinds
}
export interface Meta {
content: string;
images: string[];
videos: string[];
events: string[];
mentions: string[];
hashtags: string[];
}
export interface NostrEvent {
id: string;
pubkey: string;
@@ -36,6 +45,7 @@ export interface NostrEvent {
tags: string[][];
content: string;
sig: string;
meta: Meta;
}
export interface EventWithReplies extends NostrEvent {

57
packages/utils/src/parser.ts
View File

@@ -1,12 +1,31 @@
import { IMAGES, VIDEOS } from "./constants";
import { Meta } from "@lume/types";
import { IMAGES, NOSTR_EVENTS, NOSTR_MENTIONS, VIDEOS } from "./constants";
import { fetch } from "@tauri-apps/plugin-http";
export function parser(content: string) {
// Get clean content
export async function parser(
content: string,
abortController?: AbortController,
) {
const words = content.split(/( |\n)/);
const urls = content.match(/(https?:\/\/\S+)/gi);
// Extract hashtags
const hashtags = words.filter((word) => word.startsWith("#"));
// Extract nostr events
const events = words.filter((word) =>
NOSTR_EVENTS.some((el) => word.startsWith(el)),
);
// Extract nostr mentions
const mentions = words.filter((word) =>
NOSTR_MENTIONS.some((el) => word.startsWith(el)),
);
// Extract images and videos from content
const images: string[] = [];
const videos: string[] = [];
let text: string = content;
if (urls) {
@@ -16,20 +35,44 @@ export function parser(content: string) {
if (IMAGES.includes(ext)) {
text = text.replace(url, "");
images.push(url);
break;
}
if (VIDEOS.includes(ext)) {
text = text.replace(url, "");
videos.push(url);
break;
}
if (urls.length <= 3) {
try {
const res = await fetch(url, {
method: "HEAD",
priority: "high",
signal: abortController.signal,
// proxy: settings.proxy;
});
if (res.headers.get("Content-Type").startsWith("image")) {
text = text.replace(url, "");
images.push(url);
break;
}
} catch {
break;
}
}
}
}
const trimContent = text.trim();
return {
content: trimContent,
const meta: Meta = {
content: text.trim(),
images,
videos,
events,
mentions,
hashtags,
};
return meta;
}