diff --git a/Cargo.lock b/Cargo.lock index 6e002b4..25f78cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -898,6 +898,7 @@ dependencies = [ "lightningcss", "log", "lol_html", + "markdown", "markup5ever", "markup5ever_rcdom", "mime", @@ -1496,6 +1497,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "markdown" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5cab8f2cadc416a82d2e783a1946388b31654d391d1c7d92cc1f03e295b1deb" +dependencies = [ + "unicode-id", +] + [[package]] name = "markup5ever" version = "0.12.1" @@ -3049,6 +3059,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df77b101bcc4ea3d78dafc5ad7e4f58ceffe0b2b16bf446aeb50b6cb4157656" +[[package]] +name = "unicode-id" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ba288e709927c043cbe476718d37be306be53fb1fafecd0dbe36d072be2580" + [[package]] name = "unicode-ident" version = "1.0.17" diff --git a/Cargo.toml b/Cargo.toml index dc51f98..c3277ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,3 +47,4 @@ clap = { version = "4", features = ["derive"] } simple_logger = "4" slug = "0.1" rpassword = "7.3.1" +markdown = "1.0.0" diff --git a/src/cached.rs b/src/cached.rs index 1736171..902b534 100644 --- a/src/cached.rs +++ b/src/cached.rs @@ -182,6 +182,7 @@ impl Continuity { pub async fn get_cached( id: u64, invalidate_cache: bool, + invalidate_post_cache: bool, ) -> Result>, Box> { let board = match Board::get_cached(id, invalidate_cache).await? { Ok(board) => board, @@ -192,10 +193,13 @@ impl Continuity { let mut threads = vec![]; for p in board_posts { log::info!("Downloading post {} - {}", p.id, &p.subject); - let thread = match Thread::get_cached(p.id, invalidate_cache).await? { - Ok(thread) => thread, - Err(e) => return Ok(Err(e)), - }; + let thread = + match Thread::get_cached(p.id, invalidate_cache || invalidate_post_cache) + .await? + { + Ok(thread) => thread, + Err(e) => return Ok(Err(e)), + }; threads.push(thread); } threads diff --git a/src/generate/mod.rs b/src/generate/mod.rs index 09b615b..4982e54 100644 --- a/src/generate/mod.rs +++ b/src/generate/mod.rs @@ -15,12 +15,71 @@ use super::Thread; const STYLE: &str = include_str!("book.css"); +const MD_OPTIONS: markdown::Options = markdown::Options { + parse: markdown::ParseOptions { + constructs: markdown::Constructs { + attention: true, + autolink: true, + block_quote: true, + character_escape: true, + character_reference: true, + code_indented: false, + code_fenced: true, + code_text: true, + definition: false, + frontmatter: false, + gfm_autolink_literal: true, + gfm_footnote_definition: false, + gfm_label_start_footnote: false, + gfm_strikethrough: true, + gfm_table: true, + gfm_task_list_item: false, + hard_break_escape: false, + hard_break_trailing: true, + heading_atx: true, + heading_setext: true, + html_flow: true, + html_text: true, + label_start_image: true, + label_start_link: true, + label_end: true, + list_item: true, + math_flow: false, + math_text: false, + mdx_esm: false, + mdx_expression_flow: false, + mdx_expression_text: false, + mdx_jsx_flow: false, + mdx_jsx_text: false, + thematic_break: true, + }, + gfm_strikethrough_single_tilde: false, + math_text_single_dollar: false, + mdx_expression_parse: None, + mdx_esm_parse: None, + }, + compile: markdown::CompileOptions { + allow_any_img_src: true, + allow_dangerous_html: true, + allow_dangerous_protocol: false, + default_line_ending: markdown::LineEnding::LineFeed, + gfm_footnote_back_label: None, + gfm_footnote_clobber_prefix: None, + gfm_footnote_label_attributes: None, + gfm_footnote_label_tag_name: None, + gfm_footnote_label: None, + gfm_task_list_item_checkable: false, + gfm_tagfilter: true, + }, +}; + #[derive(Debug, Clone, Copy, Default)] pub struct Options { pub text_to_speech: bool, pub flatten_details: bool, pub jpeg: bool, pub resize_icons: Option, + pub simple_markdown_detection: bool, } fn raw_title_page(post: &Post, reply_count: usize) -> String { @@ -215,7 +274,22 @@ fn content_block( let reply_id = reply_id .map(|id| format!(r##" reply-id="{id}""##)) .unwrap_or_default(); + if options.simple_markdown_detection && !content.starts_with("

") { + let markdownified_content = markdown::to_html_with_options(content, &MD_OPTIONS).unwrap(); + return format!( + r##" + +

+
+ {image} + {caption} +
+ {markdownified_content} +
+ "## + ); + } format!( r##" diff --git a/src/main.rs b/src/main.rs index aa0883c..52f1fb0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -54,6 +54,10 @@ struct CliOptions { #[clap(long)] use_cache: bool, + /// Don't reuse data for individual posts, but still otherwise use the cache according to use_cache option. + #[clap(long)] + invalidate_post_cache: bool, + /// Simplify character and user names to improve text-to-speech output. #[clap(long)] text_to_speech: bool, @@ -71,6 +75,10 @@ struct CliOptions { #[clap(long)] jpeg: bool, + /// Uses a very simple detection-method to check for posts and replies written in Markdown. + #[clap(long)] + simple_markdown_detection: bool, + /// When inlining icons into the epub file, this will scale all icon images above the provided width down to that width. /// Defaults to "100" if no value is provided. /// (Does not affect SVGs or non-icon images.) @@ -160,9 +168,11 @@ async fn main() { let CliOptions { use_cache, + invalidate_post_cache, text_to_speech, flatten_details, jpeg, + simple_markdown_detection, resize_icons, output_dir, output_dir_layout, @@ -191,6 +201,7 @@ async fn main() { }, jpeg, resize_icons, + simple_markdown_detection, }; let html_options = Options { text_to_speech, @@ -200,12 +211,13 @@ async fn main() { }, jpeg, resize_icons, + simple_markdown_detection, }; match command { Command::Post { post_id, .. } => { log::info!("Downloading post {post_id}"); - let thread = Thread::get_cached(post_id, !use_cache) + let thread = Thread::get_cached(post_id, !use_cache || invalidate_post_cache) .await .unwrap() .unwrap(); @@ -251,7 +263,7 @@ async fn main() { .. } => { log::info!("Downloading board/continuity {board_id}..."); - let continuity = Continuity::get_cached(board_id, !use_cache) + let continuity = Continuity::get_cached(board_id, !use_cache, invalidate_post_cache) .await .unwrap() .unwrap(); @@ -297,7 +309,7 @@ async fn main() { } log::info!("Downloading board/continuity {board_id}..."); - let continuity = Continuity::get_cached(board_id, !use_cache) + let continuity = Continuity::get_cached(board_id, !use_cache, invalidate_post_cache) .await .unwrap() .unwrap();