From c42c78fc983b4e8acf32f1298061bab41eac1ca6 Mon Sep 17 00:00:00 2001 From: Ren Amamiya <123083837+reyamir@users.noreply.github.com> Date: Mon, 14 Aug 2023 09:03:58 +0700 Subject: [PATCH] clean up and refactor open graph --- package.json | 1 - pnpm-lock.yaml | 65 ---- src-tauri/Cargo.lock | 156 ++++++++- src-tauri/Cargo.toml | 1 + src-tauri/src/main.rs | 70 ++++- src-tauri/tauri.conf.json | 3 +- src/libs/ndk/{cache.tsx => cache.ts} | 14 + src/libs/openGraph.tsx | 452 --------------------------- src/libs/{storage.tsx => storage.ts} | 130 +------- src/shared/notes/preview/link.tsx | 6 +- src/utils/hooks/useOpenGraph.tsx | 5 +- src/utils/types.d.ts | 7 + 12 files changed, 256 insertions(+), 654 deletions(-) rename src/libs/ndk/{cache.tsx => cache.ts} (78%) delete mode 100644 src/libs/openGraph.tsx rename src/libs/{storage.tsx => storage.ts} (77%) diff --git a/package.json b/package.json index 447b5c48..955b530b 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,6 @@ "@tiptap/starter-kit": "^2.0.4", "@tiptap/suggestion": "^2.0.4", "@void-cat/api": "^1.0.7", - "cheerio": "1.0.0-rc.12", "dayjs": "^1.11.9", "destr": "^1.2.2", "get-urls": "^11.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 651469cf..13b2fca6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -109,9 +109,6 @@ dependencies: '@void-cat/api': specifier: ^1.0.7 version: 1.0.7 - cheerio: - specifier: 1.0.0-rc.12 - version: 1.0.0-rc.12 dayjs: specifier: ^1.11.9 version: 1.11.9 @@ -2763,10 +2760,6 @@ packages: engines: {node: '>=8'} dev: true - /boolbase@1.0.0: - resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} - dev: false - /brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} dependencies: @@ -2888,30 +2881,6 @@ packages: resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} dev: false - /cheerio-select@2.1.0: - resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} - dependencies: - boolbase: 1.0.0 - css-select: 5.1.0 - css-what: 6.1.0 - domelementtype: 2.3.0 - domhandler: 5.0.3 - domutils: 3.1.0 - dev: false - - /cheerio@1.0.0-rc.12: - resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} - engines: {node: '>= 6'} - dependencies: - cheerio-select: 2.1.0 - dom-serializer: 2.0.0 - domhandler: 5.0.3 - domutils: 3.1.0 - htmlparser2: 8.0.2 - parse5: 7.1.2 - parse5-htmlparser2-tree-adapter: 7.0.0 - dev: false - /chokidar@3.5.3: resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==} engines: {node: '>= 8.10.0'} @@ -3060,21 +3029,6 @@ packages: shebang-command: 2.0.0 which: 2.0.2 - /css-select@5.1.0: - resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} - dependencies: - boolbase: 1.0.0 - css-what: 6.1.0 - domhandler: 5.0.3 - domutils: 3.1.0 - nth-check: 2.1.1 - dev: false - - /css-what@6.1.0: - resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} - engines: {node: '>= 6'} - dev: false - /cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} @@ -5439,12 +5393,6 @@ packages: set-blocking: 2.0.0 dev: false - /nth-check@2.1.1: - resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} - dependencies: - boolbase: 1.0.0 - dev: false - /object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -5620,19 +5568,6 @@ packages: lines-and-columns: 1.2.4 dev: false - /parse5-htmlparser2-tree-adapter@7.0.0: - resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} - dependencies: - domhandler: 5.0.3 - parse5: 7.1.2 - dev: false - - /parse5@7.1.2: - resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} - dependencies: - entities: 4.5.0 - dev: false - /parseley@0.12.1: resolution: {integrity: sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==} dependencies: diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index e0c3c7c6..97c844dd 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -1123,6 +1123,36 @@ dependencies = [ "cipher 0.3.0", ] +[[package]] +name = "curl" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" +dependencies = [ + "curl-sys", + "libc", + "openssl-probe", + "openssl-sys", + "schannel", + "socket2 0.4.9", + "winapi", +] + +[[package]] +name = "curl-sys" +version = "0.4.65+curl-8.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "961ba061c9ef2fe34bbd12b807152d96f0badd2bebe7b90ce6c8c8b7572a0986" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "winapi", +] + [[package]] name = "curve25519-dalek" version = "3.2.0" @@ -2148,7 +2178,21 @@ checksum = "e5c13fb08e5d4dfc151ee5e88bae63f7773d61852f3bdc73c9f4b9e1bde03148" dependencies = [ "log", "mac", - "markup5ever", + "markup5ever 0.10.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "html5ever" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" +dependencies = [ + "log", + "mac", + "markup5ever 0.11.0", "proc-macro2", "quote", "syn 1.0.109", @@ -2538,7 +2582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ea8e9c6e031377cff82ee3001dc8026cdf431ed4e2e6b51f98ab8c73484a358" dependencies = [ "cssparser", - "html5ever", + "html5ever 0.25.2", "matches", "selectors", ] @@ -2621,6 +2665,18 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-sys" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "line-wrap" version = "0.1.1" @@ -2701,6 +2757,7 @@ dependencies = [ "tauri-plugin-updater", "tauri-plugin-upload", "tauri-plugin-window", + "webpage", "window-vibrancy", ] @@ -2740,12 +2797,38 @@ checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd" dependencies = [ "log", "phf 0.8.0", - "phf_codegen", + "phf_codegen 0.8.0", "string_cache", "string_cache_codegen", "tendril", ] +[[package]] +name = "markup5ever" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" +dependencies = [ + "log", + "phf 0.10.1", + "phf_codegen 0.10.0", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "markup5ever_rcdom" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9521dd6750f8e80ee6c53d65e2e4656d7de37064f3a7a5d2d11d05df93839c2" +dependencies = [ + "html5ever 0.26.0", + "markup5ever 0.11.0", + "tendril", + "xml5ever", +] + [[package]] name = "matchers" version = "0.1.0" @@ -3123,6 +3206,24 @@ dependencies = [ "pathdiff", ] +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-stream" version = "0.2.0" @@ -3297,6 +3398,16 @@ dependencies = [ "phf_shared 0.8.0", ] +[[package]] +name = "phf_codegen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", +] + [[package]] name = "phf_generator" version = "0.8.0" @@ -4034,6 +4145,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -4069,7 +4189,7 @@ dependencies = [ "log", "matches", "phf 0.8.0", - "phf_codegen", + "phf_codegen 0.8.0", "precomputed-hash", "servo_arc", "smallvec", @@ -5308,7 +5428,7 @@ dependencies = [ "dunce", "glob", "heck", - "html5ever", + "html5ever 0.25.2", "infer", "json-patch", "kuchiki", @@ -5949,6 +6069,19 @@ dependencies = [ "system-deps", ] +[[package]] +name = "webpage" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8598785beeb5af95abe95e7bb20c7e747d1188347080d6811d5a56d2b9a5f368" +dependencies = [ + "curl", + "html5ever 0.26.0", + "markup5ever_rcdom", + "serde", + "serde_json", +] + [[package]] name = "webpki-roots" version = "0.24.0" @@ -6385,7 +6518,7 @@ dependencies = [ "gio", "glib", "gtk", - "html5ever", + "html5ever 0.25.2", "http", "javascriptcore-rs", "kuchiki", @@ -6481,6 +6614,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "xml5ever" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4034e1d05af98b51ad7214527730626f019682d797ba38b51689212118d8e650" +dependencies = [ + "log", + "mac", + "markup5ever 0.11.0", +] + [[package]] name = "zbus" version = "3.14.1" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 8adb9bfa..0d4d2fb2 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -42,6 +42,7 @@ sqlx-cli = { version = "0.7.0", default-features = false, features = [ ] } rust-argon2 = "1.0" rand = "0.8.5" +webpage = { version = "1.1", features = ["serde"] } [dependencies.tauri-plugin-sql] git = "https://github.com/tauri-apps/plugins-workspace" diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index bb92ee3d..ed45ae11 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -3,10 +3,13 @@ windows_subsystem = "windows" )] +use std::time::Duration; + // use rand::distributions::{Alphanumeric, DistString}; use tauri::Manager; use tauri_plugin_autostart::MacosLauncher; use tauri_plugin_sql::{Migration, MigrationKind}; +use webpage::{Webpage, WebpageOptions}; use window_vibrancy::{apply_mica, apply_vibrancy, NSVisualEffectMaterial}; #[derive(Clone, serde::Serialize)] @@ -15,6 +18,71 @@ struct Payload { cwd: String, } +#[derive(serde::Serialize)] +struct OpenGraphResponse { + title: String, + description: String, + url: String, + image: String, +} + +async fn fetch_opengraph(url: String) -> OpenGraphResponse { + let options = WebpageOptions { + allow_insecure: false, + max_redirections: 3, + timeout: Duration::from_secs(15), + useragent: "lume - desktop app".to_string(), + ..Default::default() + }; + + let result = match Webpage::from_url(&url, options) { + Ok(webpage) => webpage, + Err(_) => { + return OpenGraphResponse { + title: "".to_string(), + description: "".to_string(), + url: "".to_string(), + image: "".to_string(), + } + } + }; + + let html = result.html; + + return OpenGraphResponse { + title: html + .opengraph + .properties + .get("title") + .cloned() + .unwrap_or_default(), + description: html + .opengraph + .properties + .get("description") + .cloned() + .unwrap_or_default(), + url: html + .opengraph + .properties + .get("url") + .cloned() + .unwrap_or_default(), + image: html + .opengraph + .images + .get(0) + .and_then(|i| Some(i.url.clone())) + .unwrap_or_default(), + }; +} + +#[tauri::command] +async fn opengraph(url: String) -> OpenGraphResponse { + let result = fetch_opengraph(url).await; + return result; +} + #[tauri::command] async fn close_splashscreen(window: tauri::Window) { // Close splashscreen @@ -179,7 +247,7 @@ fn main() { Ok(()) }) - .invoke_handler(tauri::generate_handler![close_splashscreen]) + .invoke_handler(tauri::generate_handler![close_splashscreen, opengraph]) .run(tauri::generate_context!()) .expect("error while running tauri application"); } diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index e674efea..80ad772f 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -67,7 +67,8 @@ "exceptionDomain": "", "frameworks": [], "providerShortName": null, - "signingIdentity": null + "signingIdentity": null, + "minimumSystemVersion": "10.15.0" }, "resources": [], "shortDescription": "", diff --git a/src/libs/ndk/cache.tsx b/src/libs/ndk/cache.ts similarity index 78% rename from src/libs/ndk/cache.tsx rename to src/libs/ndk/cache.ts index cdbe4993..fed7815a 100644 --- a/src/libs/ndk/cache.tsx +++ b/src/libs/ndk/cache.ts @@ -31,12 +31,26 @@ export default class TauriAdapter implements NDKCacheAdapter { const event = await this.store.get(result as string); if (event) { + console.log('cache hit: ', result); const ndkEvent = new NDKEvent(subscription.ndk, JSON.parse(event as string)); subscription.eventReceived(ndkEvent, undefined, true); } } } } + + if (filter.ids) { + for (const id of filter.ids) { + const key = id; + const event = await this.store.get(key); + + if (event) { + console.log('cache hit: ', id); + const ndkEvent = new NDKEvent(subscription.ndk, JSON.parse(event as string)); + subscription.eventReceived(ndkEvent, undefined, true); + } + } + } } public async setEvent(event: NDKEvent): Promise { diff --git a/src/libs/openGraph.tsx b/src/libs/openGraph.tsx deleted file mode 100644 index 3ef87cdf..00000000 --- a/src/libs/openGraph.tsx +++ /dev/null @@ -1,452 +0,0 @@ -import { fetch } from '@tauri-apps/plugin-http'; -import * as cheerio from 'cheerio'; - -import { OPENGRAPH } from '@stores/constants'; - -interface ILinkPreviewOptions { - headers?: Record; - imagesPropertyType?: string; - proxyUrl?: string; - timeout?: number; - followRedirects?: `follow` | `error` | `manual`; - resolveDNSHost?: (url: string) => Promise; - handleRedirects?: (baseURL: string, forwardedURL: string) => boolean; -} - -interface IPreFetchedResource { - headers: Record; - status?: number; - imagesPropertyType?: string; - proxyUrl?: string; - url: string; - data: string; -} - -function throwOnLoopback(address: string) { - if (OPENGRAPH.REGEX_LOOPBACK.test(address)) { - throw new Error('SSRF request detected, trying to query host'); - } -} - -function metaTag(doc: cheerio.CheerioAPI, type: string, attr: string) { - const nodes = doc(`meta[${attr}='${type}']`); - return nodes.length ? nodes : null; -} - -function metaTagContent(doc: cheerio.CheerioAPI, type: string, attr: string) { - return doc(`meta[${attr}='${type}']`).attr(`content`); -} - -function getTitle(doc: cheerio.CheerioAPI) { - let title = - metaTagContent(doc, `og:title`, `property`) || - metaTagContent(doc, `og:title`, `name`); - if (!title) { - title = doc(`title`).text(); - } - return title; -} - -function getSiteName(doc: cheerio.CheerioAPI) { - const siteName = - metaTagContent(doc, `og:site_name`, `property`) || - metaTagContent(doc, `og:site_name`, `name`); - return siteName; -} - -function getDescription(doc: cheerio.CheerioAPI) { - const description = - metaTagContent(doc, `description`, `name`) || - metaTagContent(doc, `Description`, `name`) || - metaTagContent(doc, `og:description`, `property`); - return description; -} - -function getMediaType(doc: cheerio.CheerioAPI) { - const node = metaTag(doc, `medium`, `name`); - if (node) { - const content = node.attr(`content`); - return content === `image` ? `photo` : content; - } - return ( - metaTagContent(doc, `og:type`, `property`) || metaTagContent(doc, `og:type`, `name`) - ); -} - -function getImages( - doc: cheerio.CheerioAPI, - rootUrl: string, - imagesPropertyType?: string -) { - let images: string[] = []; - let nodes: cheerio.Cheerio | null; - let src: string | undefined; - let dic: Record = {}; - - const imagePropertyType = imagesPropertyType ?? `og`; - nodes = - metaTag(doc, `${imagePropertyType}:image`, `property`) || - metaTag(doc, `${imagePropertyType}:image`, `name`); - - if (nodes) { - nodes.each((_: number, node: cheerio.Element) => { - if (node.type === `tag`) { - src = node.attribs.content; - if (src) { - src = new URL(src, rootUrl).href; - images.push(src); - } - } - }); - } - - if (images.length <= 0 && !imagesPropertyType) { - src = doc(`link[rel=image_src]`).attr(`href`); - if (src) { - src = new URL(src, rootUrl).href; - images = [src]; - } else { - nodes = doc(`img`); - - if (nodes?.length) { - dic = {}; - images = []; - nodes.each((_: number, node: cheerio.Element) => { - if (node.type === `tag`) src = node.attribs.src; - if (src && !dic[src]) { - dic[src] = true; - // width = node.attribs.width; - // height = node.attribs.height; - images.push(new URL(src, rootUrl).href); - } - }); - } - } - } - - return images; -} - -function getVideos(doc: cheerio.CheerioAPI) { - const videos = []; - let nodeTypes; - let nodeSecureUrls; - let nodeType; - let nodeSecureUrl; - let video; - let videoType; - let videoSecureUrl; - let width; - let height; - let videoObj; - let index; - - const nodes = metaTag(doc, `og:video`, `property`) || metaTag(doc, `og:video`, `name`); - - if (nodes?.length) { - nodeTypes = - metaTag(doc, `og:video:type`, `property`) || metaTag(doc, `og:video:type`, `name`); - nodeSecureUrls = - metaTag(doc, `og:video:secure_url`, `property`) || - metaTag(doc, `og:video:secure_url`, `name`); - width = - metaTagContent(doc, `og:video:width`, `property`) || - metaTagContent(doc, `og:video:width`, `name`); - height = - metaTagContent(doc, `og:video:height`, `property`) || - metaTagContent(doc, `og:video:height`, `name`); - - for (index = 0; index < nodes.length; index += 1) { - const node = nodes[index]; - if (node.type === `tag`) video = node.attribs.content; - - nodeType = nodeTypes?.[index]; - if (nodeType?.type === `tag`) { - videoType = nodeType ? nodeType.attribs.content : null; - } - - nodeSecureUrl = nodeSecureUrls?.[index]; - if (nodeSecureUrl?.type === `tag`) { - videoSecureUrl = nodeSecureUrl ? nodeSecureUrl.attribs.content : null; - } - - videoObj = { - url: video, - secureUrl: videoSecureUrl, - type: videoType, - width, - height, - }; - if (videoType && videoType.indexOf(`video/`) === 0) { - videos.splice(0, 0, videoObj); - } else { - videos.push(videoObj); - } - } - } - - return videos; -} - -// returns default favicon (//hostname/favicon.ico) for a url -function getDefaultFavicon(rootUrl: string) { - return `${new URL(rootUrl).origin}/favicon.ico`; -} - -// returns an array of URLs to favicon images -function getFavicons(doc: cheerio.CheerioAPI, rootUrl: string) { - const images = []; - let nodes: cheerio.Cheerio | never[] = []; - let src: string | undefined; - - const relSelectors = [`rel=icon`, `rel="shortcut icon"`, `rel=apple-touch-icon`]; - - relSelectors.forEach((relSelector) => { - // look for all icon tags - nodes = doc(`link[${relSelector}]`); - - // collect all images from icon tags - if (nodes.length) { - nodes.each((_: number, node: cheerio.Element) => { - if (node.type === `tag`) src = node.attribs.href; - if (src) { - src = new URL(src, rootUrl).href; - images.push(src); - } - }); - } - }); - - // if no icon images, use default favicon location - if (images.length <= 0) { - images.push(getDefaultFavicon(rootUrl)); - } - - return images; -} - -function parseImageResponse(url: string, contentType: string) { - return { - url, - mediaType: `image`, - contentType, - favicons: [getDefaultFavicon(url)], - }; -} - -function parseAudioResponse(url: string, contentType: string) { - return { - url, - mediaType: `audio`, - contentType, - favicons: [getDefaultFavicon(url)], - }; -} - -function parseVideoResponse(url: string, contentType: string) { - return { - url, - mediaType: `video`, - contentType, - favicons: [getDefaultFavicon(url)], - }; -} - -function parseApplicationResponse(url: string, contentType: string) { - return { - url, - mediaType: `application`, - contentType, - favicons: [getDefaultFavicon(url)], - }; -} - -function parseTextResponse( - body: string, - url: string, - options: ILinkPreviewOptions = {}, - contentType?: string -) { - const doc = cheerio.load(body); - - return { - url, - title: getTitle(doc), - siteName: getSiteName(doc), - description: getDescription(doc), - mediaType: getMediaType(doc) || `website`, - contentType, - images: getImages(doc, url, options.imagesPropertyType), - videos: getVideos(doc), - favicons: getFavicons(doc, url), - }; -} - -function parseUnknownResponse( - body: string, - url: string, - options: ILinkPreviewOptions = {}, - contentType?: string -) { - return parseTextResponse(body, url, options, contentType); -} - -function parseResponse(response: IPreFetchedResource, options?: ILinkPreviewOptions) { - try { - let contentType = response.headers[`content-type`]; - // console.warn(`original content type`, contentType); - if (contentType?.indexOf(`;`)) { - // eslint-disable-next-line prefer-destructuring - contentType = contentType.split(`;`)[0]; - // console.warn(`splitting content type`, contentType); - } - - if (!contentType) { - return parseUnknownResponse(response.data, response.url, options); - } - - if ((contentType as any) instanceof Array) { - // eslint-disable-next-line no-param-reassign, prefer-destructuring - contentType = contentType[0]; - } - - // parse response depending on content type - if (OPENGRAPH.REGEX_CONTENT_TYPE_IMAGE.test(contentType)) { - return parseImageResponse(response.url, contentType); - } - if (OPENGRAPH.REGEX_CONTENT_TYPE_AUDIO.test(contentType)) { - return parseAudioResponse(response.url, contentType); - } - if (OPENGRAPH.REGEX_CONTENT_TYPE_VIDEO.test(contentType)) { - return parseVideoResponse(response.url, contentType); - } - if (OPENGRAPH.REGEX_CONTENT_TYPE_TEXT.test(contentType)) { - const htmlString = response.data; - return parseTextResponse(htmlString, response.url, options, contentType); - } - if (OPENGRAPH.REGEX_CONTENT_TYPE_APPLICATION.test(contentType)) { - return parseApplicationResponse(response.url, contentType); - } - const htmlString = response.data; - return parseUnknownResponse(htmlString, response.url, options); - } catch (e) { - throw new Error( - `link-preview-js could not fetch link information ${(e as any).toString()}` - ); - } -} - -/** - * Parses the text, extracts the first link it finds and does a HTTP request - * to fetch the website content, afterwards it tries to parse the internal HTML - * and extract the information via meta tags - * @param text string, text to be parsed - * @param options ILinkPreviewOptions - */ -export async function getLinkPreview(text: string, options?: ILinkPreviewOptions) { - if (!text || typeof text !== `string`) { - throw new Error(`link-preview-js did not receive a valid url or text`); - } - - const detectedUrl = text - .replace(/\n/g, ` `) - .split(` `) - .find((token) => OPENGRAPH.REGEX_VALID_URL.test(token)); - - if (!detectedUrl) { - throw new Error(`link-preview-js did not receive a valid a url or text`); - } - - if (options?.followRedirects === `manual` && !options?.handleRedirects) { - throw new Error( - `link-preview-js followRedirects is set to manual, but no handleRedirects function was provided` - ); - } - - if (options?.resolveDNSHost) { - const resolvedUrl = await options.resolveDNSHost(detectedUrl); - - throwOnLoopback(resolvedUrl); - } - - const timeout = options?.timeout ?? 3000; // 3 second timeout default - const controller = new AbortController(); - const timeoutCounter = setTimeout(() => controller.abort(), timeout); - - const fetchOptions = { - headers: options?.headers ?? {}, - redirect: options?.followRedirects ?? `error`, - signal: controller.signal, - }; - - const fetchUrl = options?.proxyUrl ? options.proxyUrl.concat(detectedUrl) : detectedUrl; - - // Seems like fetchOptions type definition is out of date - // https://github.com/node-fetch/node-fetch/issues/741 - let response = await fetch(fetchUrl, fetchOptions as any).catch((e) => { - if (e.name === `AbortError`) { - throw new Error(`Request timeout`); - } - - clearTimeout(timeoutCounter); - throw e; - }); - - if ( - response.status > 300 && - response.status < 309 && - fetchOptions.redirect === `manual` && - options?.handleRedirects - ) { - const forwardedUrl = response.headers.get(`location`) || ``; - - if (!options.handleRedirects(fetchUrl, forwardedUrl)) { - throw new Error(`link-preview-js could not handle redirect`); - } - - if (options?.resolveDNSHost) { - const resolvedUrl = await options.resolveDNSHost(forwardedUrl); - - throwOnLoopback(resolvedUrl); - } - - response = await fetch(forwardedUrl, fetchOptions as any); - } - - clearTimeout(timeoutCounter); - - const headers: Record = {}; - response.headers.forEach((header, key) => { - headers[key] = header; - }); - - const normalizedResponse: IPreFetchedResource = { - url: options?.proxyUrl ? response.url.replace(options.proxyUrl, ``) : response.url, - headers, - data: await response.text(), - }; - - return parseResponse(normalizedResponse, options); -} - -/** - * Skip the library fetching the website for you, instead pass a response object - * from whatever source you get and use the internal parsing of the HTML to return - * the necessary information - * @param response Preview Response - * @param options IPreviewLinkOptions - */ -export async function getPreviewFromContent( - response: IPreFetchedResource, - options?: ILinkPreviewOptions -) { - if (!response || typeof response !== `object`) { - throw new Error(`link-preview-js did not receive a valid response object`); - } - - if (!response.url) { - throw new Error(`link-preview-js did not receive a valid response object`); - } - - return parseResponse(response, options); -} diff --git a/src/libs/storage.tsx b/src/libs/storage.ts similarity index 77% rename from src/libs/storage.tsx rename to src/libs/storage.ts index ad747e88..41600df1 100644 --- a/src/libs/storage.tsx +++ b/src/libs/storage.ts @@ -21,7 +21,11 @@ export async function connect(): Promise { if (db) { return db; } - db = await Database.load('sqlite:lume.db'); + try { + db = await Database.load('sqlite:lume.db'); + } catch (e) { + throw new Error('Failed to connect to database, error: ', e); + } return db; } @@ -44,26 +48,12 @@ export async function getActiveAccount() { } } -// get all accounts -export async function getAccounts() { - const db = await connect(); - const result: Array = await db.select( - 'SELECT * FROM accounts WHERE is_active = 0 ORDER BY created_at DESC;' - ); - return result; -} - // create account -export async function createAccount( - npub: string, - pubkey: string, - follows?: string[][], - is_active?: number -) { +export async function createAccount(npub: string, pubkey: string, follows?: string[][]) { const db = await connect(); const res = await db.execute( 'INSERT OR IGNORE INTO accounts (npub, pubkey, privkey, follows, is_active) VALUES (?, ?, ?, ?, ?);', - [npub, pubkey, 'privkey is stored in secure storage', follows || '', is_active || 0] + [npub, pubkey, 'privkey is stored in secure storage', follows || '', 1] ); if (res) { await createWidget( @@ -86,13 +76,6 @@ export async function updateAccount(column: string, value: string | string[]) { ]); } -// count total notes -export async function countTotalChannels() { - const db = await connect(); - const result = await db.select('SELECT COUNT(*) AS "total" FROM channels;'); - return result[0]; -} - // count total notes export async function countTotalNotes() { const db = await connect(); @@ -127,21 +110,6 @@ export async function getNotes(limit: number, offset: number) { return notes; } -// get all notes by pubkey -export async function getNotesByPubkey(pubkey: string) { - const db = await connect(); - - const query: LumeEvent[] = await db.select( - `SELECT * FROM notes WHERE pubkey == "${pubkey}" AND kind IN (1, 6, 1063) GROUP BY parent_id ORDER BY created_at DESC;` - ); - - query.forEach( - (el) => (el.tags = typeof el.tags === 'string' ? destr(el.tags) : el.tags) - ); - - return query; -} - // get all notes by authors export async function getNotesByAuthors(authors: string, limit: number, offset: number) { const db = await connect(); @@ -228,90 +196,6 @@ export async function createReplyNote( ); } -// get all pubkeys in db -export async function getAllPubkeys() { - const db = await connect(); - const notes: any = await db.select('SELECT DISTINCT pubkey FROM notes'); - const replies: any = await db.select('SELECT DISTINCT pubkey FROM replies'); - const chats: any = await db.select('SELECT DISTINCT sender_pubkey FROM chats'); - return [...notes, ...replies, ...chats]; -} - -// get all channels -export async function getChannels() { - const db = await connect(); - const result: any = await db.select('SELECT * FROM channels ORDER BY created_at DESC;'); - return result; -} - -// get channel by id -export async function getChannel(id: string) { - const db = await connect(); - const result = await db.select(`SELECT * FROM channels WHERE event_id = "${id}";`); - return result[0]; -} - -// create channel -export async function createChannel( - event_id: string, - pubkey: string, - name: string, - picture: string, - about: string, - created_at: number -) { - const db = await connect(); - return await db.execute( - 'INSERT OR IGNORE INTO channels (event_id, pubkey, name, picture, about, created_at) VALUES (?, ?, ?, ?, ?, ?);', - [event_id, pubkey, name, picture, about, created_at] - ); -} - -// update channel metadata -export async function updateChannelMetadata(event_id: string, value: string) { - const db = await connect(); - const data = JSON.parse(value); - - return await db.execute( - 'UPDATE channels SET name = ?, picture = ?, about = ? WHERE event_id = ?;', - [data.name, data.picture, data.about, event_id] - ); -} - -// create channel messages -export async function createChannelMessage( - channel_id: string, - event_id: string, - pubkey: string, - kind: number, - content: string, - tags: string[][], - created_at: number -) { - const db = await connect(); - return await db.execute( - 'INSERT OR IGNORE INTO channel_messages (channel_id, event_id, pubkey, kind, content, tags, created_at) VALUES (?, ?, ?, ?, ?, ?, ?);', - [channel_id, event_id, pubkey, kind, content, tags, created_at] - ); -} - -// get channel messages by channel id -export async function getChannelMessages(channel_id: string) { - const db = await connect(); - return await db.select( - `SELECT * FROM channel_messages WHERE channel_id = "${channel_id}" ORDER BY created_at ASC;` - ); -} - -// get channel users -export async function getChannelUsers(channel_id: string) { - const db = await connect(); - const result: any = await db.select( - `SELECT DISTINCT pubkey FROM channel_messages WHERE channel_id = "${channel_id}";` - ); - return result; -} - // get all chats by pubkey export async function getChats() { const db = await connect(); diff --git a/src/shared/notes/preview/link.tsx b/src/shared/notes/preview/link.tsx index 42b374d4..3ad7eaef 100644 --- a/src/shared/notes/preview/link.tsx +++ b/src/shared/notes/preview/link.tsx @@ -35,10 +35,10 @@ export function LinkPreview({ urls }: { urls: string[] }) { ) : ( <> - {data.images?.[0] && ( + {data.image && ( {urls[0]} diff --git a/src/utils/hooks/useOpenGraph.tsx b/src/utils/hooks/useOpenGraph.tsx index fc3d9a46..bbdc527f 100644 --- a/src/utils/hooks/useOpenGraph.tsx +++ b/src/utils/hooks/useOpenGraph.tsx @@ -1,12 +1,13 @@ import { useQuery } from '@tanstack/react-query'; +import { invoke } from '@tauri-apps/api/tauri'; -import { getLinkPreview } from '@libs/openGraph'; +import { Opengraph } from '@utils/types'; export function useOpenGraph(url: string) { const { status, data, error, isFetching } = useQuery( ['preview', url], async () => { - const res = await getLinkPreview(url); + const res: Opengraph = await invoke('opengraph', { url }); if (!res) { throw new Error('fetch preview failed'); } diff --git a/src/utils/types.d.ts b/src/utils/types.d.ts index 8fbbb045..470774e7 100644 --- a/src/utils/types.d.ts +++ b/src/utils/types.d.ts @@ -62,3 +62,10 @@ export interface Relays { relay: string; purpose?: string; } + +export interface Opengraph { + url: string; + title?: string; + description?: string; + image?: string; +}