From a5e6f7942c6baf92cfb8e2cb591259d2bb2931d5 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 13 Jan 2021 10:39:13 -0500 Subject: [PATCH] Rename MarkdownParser to MarkdownProcessor::Parser (#12248) * Rename MarkdownParser to MarkdownProcessor::Parse * Wake up, Travis * Fix typo --- app/controllers/articles_controller.rb | 2 +- app/controllers/comments_controller.rb | 2 +- app/models/article.rb | 2 +- app/models/badge_achievement.rb | 2 +- app/models/comment.rb | 4 +- app/models/event.rb | 2 +- app/models/listing.rb | 2 +- app/models/message.rb | 2 +- app/models/organization.rb | 2 +- app/models/page.rb | 2 +- app/models/poll.rb | 2 +- app/models/poll_option.rb | 2 +- app/models/tag.rb | 4 +- app/models/user_subscription.rb | 2 +- app/services/markdown_parser.rb | 145 ----------------- app/services/markdown_processor/parser.rb | 147 ++++++++++++++++++ .../parser_spec.rb} | 2 +- 17 files changed, 164 insertions(+), 162 deletions(-) delete mode 100644 app/services/markdown_parser.rb create mode 100644 app/services/markdown_processor/parser.rb rename spec/services/{markdown_parser_spec.rb => markdown_processor/parser_spec.rb} (99%) diff --git a/app/controllers/articles_controller.rb b/app/controllers/articles_controller.rb index 05d0a404f..075a3f79d 100644 --- a/app/controllers/articles_controller.rb +++ b/app/controllers/articles_controller.rb @@ -87,7 +87,7 @@ class ArticlesController < ApplicationController begin fixed_body_markdown = MarkdownFixer.fix_for_preview(params[:article_body]) parsed = FrontMatterParser::Parser.new(:md).call(fixed_body_markdown) - parsed_markdown = MarkdownParser.new(parsed.content, source: Article.new, user: current_user) + parsed_markdown = MarkdownProcessor::Parser.new(parsed.content, source: Article.new, user: current_user) processed_html = parsed_markdown.finalize rescue StandardError => e @article = Article.new(body_markdown: params[:article_body]) diff --git a/app/controllers/comments_controller.rb b/app/controllers/comments_controller.rb index e17213151..1a6372e3e 100644 --- a/app/controllers/comments_controller.rb +++ b/app/controllers/comments_controller.rb @@ -204,7 +204,7 @@ class CommentsController < ApplicationController begin permitted_body_markdown = permitted_attributes(Comment)[:body_markdown] fixed_body_markdown = MarkdownFixer.fix_for_preview(permitted_body_markdown) - parsed_markdown = MarkdownParser.new(fixed_body_markdown, source: Comment.new, user: current_user) + parsed_markdown = MarkdownProcessor::Parser.new(fixed_body_markdown, source: Comment.new, user: current_user) processed_html = parsed_markdown.finalize rescue StandardError => e processed_html = "

😔 There was an error in your markdown

#{e}

" diff --git a/app/models/article.rb b/app/models/article.rb index 0a42fd62e..515cc992c 100644 --- a/app/models/article.rb +++ b/app/models/article.rb @@ -416,7 +416,7 @@ class Article < ApplicationRecord def evaluate_markdown fixed_body_markdown = MarkdownFixer.fix_all(body_markdown || "") parsed = FrontMatterParser::Parser.new(:md).call(fixed_body_markdown) - parsed_markdown = MarkdownParser.new(parsed.content, source: self, user: user) + parsed_markdown = MarkdownProcessor::Parser.new(parsed.content, source: self, user: user) self.reading_time = parsed_markdown.calculate_reading_time self.processed_html = parsed_markdown.finalize diff --git a/app/models/badge_achievement.rb b/app/models/badge_achievement.rb index ca703abda..a0aef90da 100644 --- a/app/models/badge_achievement.rb +++ b/app/models/badge_achievement.rb @@ -24,7 +24,7 @@ class BadgeAchievement < ApplicationRecord def render_rewarding_context_message_html return unless rewarding_context_message_markdown - parsed_markdown = MarkdownParser.new(rewarding_context_message_markdown) + parsed_markdown = MarkdownProcessor::Parser.new(rewarding_context_message_markdown) html = parsed_markdown.finalize final_html = ActionController::Base.helpers.sanitize( html, diff --git a/app/models/comment.rb b/app/models/comment.rb index ebd6ab8ab..9b573eeaf 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -99,7 +99,7 @@ class Comment < ApplicationRecord end def custom_css - MarkdownParser.new(body_markdown).tags_used.map do |tag| + MarkdownProcessor::Parser.new(body_markdown).tags_used.map do |tag| Rails.application.assets["ltags/#{tag}.css"].to_s end.join end @@ -163,7 +163,7 @@ class Comment < ApplicationRecord def evaluate_markdown fixed_body_markdown = MarkdownFixer.fix_for_comment(body_markdown) - parsed_markdown = MarkdownParser.new(fixed_body_markdown, source: self, user: user) + parsed_markdown = MarkdownProcessor::Parser.new(fixed_body_markdown, source: self, user: user) self.processed_html = parsed_markdown.finalize(link_attributes: { rel: "nofollow" }) wrap_timestamps_if_video_present! if commentable shorten_urls! diff --git a/app/models/event.rb b/app/models/event.rb index f059e3a79..00cdd37c3 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -23,7 +23,7 @@ class Event < ApplicationRecord private def evaluate_markdown - self.description_html = MarkdownParser.new(description_markdown).evaluate_markdown + self.description_html = MarkdownProcessor::Parser.new(description_markdown).evaluate_markdown end def end_time_after_start diff --git a/app/models/listing.rb b/app/models/listing.rb index 1d858d2ce..4b832cd08 100644 --- a/app/models/listing.rb +++ b/app/models/listing.rb @@ -69,7 +69,7 @@ class Listing < ApplicationRecord private def evaluate_markdown - self.processed_html = MarkdownParser.new(body_markdown).evaluate_listings_markdown + self.processed_html = MarkdownProcessor::Parser.new(body_markdown).evaluate_listings_markdown end def modify_inputs diff --git a/app/models/message.rb b/app/models/message.rb index 5760e2618..3631908b1 100644 --- a/app/models/message.rb +++ b/app/models/message.rb @@ -45,7 +45,7 @@ class Message < ApplicationRecord end def evaluate_markdown - html = MarkdownParser.new(message_markdown).evaluate_markdown + html = MarkdownProcessor::Parser.new(message_markdown).evaluate_markdown html = target_blank_links(html) html = append_rich_links(html) html = wrap_mentions_with_links(html) diff --git a/app/models/organization.rb b/app/models/organization.rb index d06fa1006..a8e3693c7 100644 --- a/app/models/organization.rb +++ b/app/models/organization.rb @@ -126,7 +126,7 @@ class Organization < ApplicationRecord private def evaluate_markdown - self.cta_processed_html = MarkdownParser.new(cta_body_markdown).evaluate_limited_markdown + self.cta_processed_html = MarkdownProcessor::Parser.new(cta_body_markdown).evaluate_limited_markdown end def remove_at_from_usernames diff --git a/app/models/page.rb b/app/models/page.rb index 18e37a6ff..2dcaf5774 100644 --- a/app/models/page.rb +++ b/app/models/page.rb @@ -27,7 +27,7 @@ class Page < ApplicationRecord def evaluate_markdown if body_markdown.present? - parsed_markdown = MarkdownParser.new(body_markdown) + parsed_markdown = MarkdownProcessor::Parser.new(body_markdown) self.processed_html = parsed_markdown.finalize else self.processed_html = body_html diff --git a/app/models/poll.rb b/app/models/poll.rb index ab274f7ee..c8904690f 100644 --- a/app/models/poll.rb +++ b/app/models/poll.rb @@ -31,6 +31,6 @@ class Poll < ApplicationRecord end def evaluate_markdown - self.prompt_html = MarkdownParser.new(prompt_markdown).evaluate_inline_limited_markdown + self.prompt_html = MarkdownProcessor::Parser.new(prompt_markdown).evaluate_inline_limited_markdown end end diff --git a/app/models/poll_option.rb b/app/models/poll_option.rb index ea3a18f09..d0eb57aaf 100644 --- a/app/models/poll_option.rb +++ b/app/models/poll_option.rb @@ -12,6 +12,6 @@ class PollOption < ApplicationRecord private def evaluate_markdown - self.processed_html = MarkdownParser.new(markdown).evaluate_inline_limited_markdown + self.processed_html = MarkdownProcessor::Parser.new(markdown).evaluate_inline_limited_markdown end end diff --git a/app/models/tag.rb b/app/models/tag.rb index 566cde913..2118e37a4 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -96,8 +96,8 @@ class Tag < ActsAsTaggableOn::Tag private def evaluate_markdown - self.rules_html = MarkdownParser.new(rules_markdown).evaluate_markdown - self.wiki_body_html = MarkdownParser.new(wiki_body_markdown).evaluate_markdown + self.rules_html = MarkdownProcessor::Parser.new(rules_markdown).evaluate_markdown + self.wiki_body_html = MarkdownProcessor::Parser.new(wiki_body_markdown).evaluate_markdown end def calculate_hotness_score diff --git a/app/models/user_subscription.rb b/app/models/user_subscription.rb index 9dbadd4d2..42cbab1b6 100644 --- a/app/models/user_subscription.rb +++ b/app/models/user_subscription.rb @@ -58,7 +58,7 @@ class UserSubscription < ApplicationRecord end def liquid_tags_used - MarkdownParser.new( + MarkdownProcessor::Parser.new( user_subscription_sourceable.body_markdown, source: user_subscription_sourceable, user: user_subscription_sourceable.user, diff --git a/app/services/markdown_parser.rb b/app/services/markdown_parser.rb deleted file mode 100644 index 00feac7e9..000000000 --- a/app/services/markdown_parser.rb +++ /dev/null @@ -1,145 +0,0 @@ -class MarkdownParser - include ApplicationHelper - - BAD_XSS_REGEX = [ - /src=["'](data|&)/i, - %r{data:text/html[,;][\sa-z0-9]*}i, - ].freeze - - WORDS_READ_PER_MINUTE = 275.0 - - def initialize(content, source: nil, user: nil) - @content = content - @source = source - @user = user - end - - def finalize(link_attributes: {}) - options = { hard_wrap: true, filter_html: false, link_attributes: link_attributes } - renderer = Redcarpet::Render::HTMLRouge.new(options) - markdown = Redcarpet::Markdown.new(renderer, Constants::Redcarpet::CONFIG) - catch_xss_attempts(@content) - escaped_content = escape_liquid_tags_in_codeblock(@content) - html = markdown.render(escaped_content) - sanitized_content = sanitize_rendered_markdown(html) - begin - liquid_tag_options = { source: @source, user: @user } - parsed_liquid = Liquid::Template.parse(sanitized_content, liquid_tag_options) - html = markdown.render(parsed_liquid.render) - rescue Liquid::SyntaxError => e - html = e.message - end - - parse_html(html) - end - - def calculate_reading_time - word_count = @content.split(/\W+/).count - (word_count / WORDS_READ_PER_MINUTE).ceil - end - - def evaluate_markdown - return if @content.blank? - - renderer = Redcarpet::Render::HTMLRouge.new(hard_wrap: true, filter_html: false) - markdown = Redcarpet::Markdown.new(renderer, Constants::Redcarpet::CONFIG) - allowed_tags = %w[strong abbr aside em p h1 h2 h3 h4 h5 h6 i u b code pre - br ul ol li small sup sub img a span hr blockquote kbd] - allowed_attributes = %w[href strong em ref rel src title alt class] - ActionController::Base.helpers.sanitize markdown.render(@content), - tags: allowed_tags, - attributes: allowed_attributes - end - - def evaluate_limited_markdown - return if @content.blank? - - renderer = Redcarpet::Render::HTMLRouge.new(hard_wrap: true, filter_html: false) - markdown = Redcarpet::Markdown.new(renderer, Constants::Redcarpet::CONFIG) - allowed_tags = %w[strong i u b em p br code] - allowed_attributes = %w[href strong em ref rel src title alt class] - ActionController::Base.helpers.sanitize markdown.render(@content), - tags: allowed_tags, - attributes: allowed_attributes - end - - def evaluate_inline_limited_markdown - return if @content.blank? - - renderer = Redcarpet::Render::HTMLRouge.new(hard_wrap: true, filter_html: false) - markdown = Redcarpet::Markdown.new(renderer, Constants::Redcarpet::CONFIG) - allowed_tags = %w[strong i u b em code] - allowed_attributes = %w[href strong em ref rel src title alt class] - ActionController::Base.helpers.sanitize markdown.render(@content), - tags: allowed_tags, - attributes: allowed_attributes - end - - def evaluate_listings_markdown - return if @content.blank? - - renderer = Redcarpet::Render::HTMLRouge.new(hard_wrap: true, filter_html: false) - markdown = Redcarpet::Markdown.new(renderer, Constants::Redcarpet::CONFIG) - allowed_tags = %w[strong abbr aside em p h4 h5 h6 i u b code pre - br ul ol li small sup sub a span hr blockquote kbd] - allowed_attributes = %w[href strong em ref rel src title alt class] - ActionController::Base.helpers.sanitize markdown.render(@content), - tags: allowed_tags, - attributes: allowed_attributes - end - - def tags_used - return [] if @content.blank? - - cleaned_parsed = escape_liquid_tags_in_codeblock(@content) - tags = [] - liquid_tag_options = { source: @source, user: @user } - Liquid::Template.parse(cleaned_parsed, liquid_tag_options).root.nodelist.each do |node| - tags << node.class if node.class.superclass.to_s == LiquidTagBase.to_s - end - tags.uniq - rescue Liquid::SyntaxError - [] - end - - def catch_xss_attempts(markdown) - return unless markdown.match?(Regexp.union(BAD_XSS_REGEX)) - - raise ArgumentError, "Invalid markdown detected" - end - - def escape_liquid_tags_in_codeblock(content) - # Escape codeblocks, code spans, and inline code - content.gsub(/[[:space:]]*`{3}.*?`{3}|`{2}.+?`{2}|`{1}.+?`{1}/m) do |codeblock| - codeblock.gsub!("{% endraw %}", "{----% endraw %----}") - codeblock.gsub!("{% raw %}", "{----% raw %----}") - if codeblock.match?(/[[:space:]]*`{3}/) - "\n{% raw %}\n#{codeblock}\n{% endraw %}\n" - else - "{% raw %}#{codeblock}{% endraw %}" - end - end - end - - private - - def parse_html(html) - return html if html.blank? - - Html::Parser - .new(html) - .remove_nested_linebreak_in_list - .prefix_all_images - .wrap_all_images_in_links - .add_control_class_to_codeblock - .add_control_panel_to_codeblock - .add_fullscreen_button_to_panel - .wrap_all_tables - .remove_empty_paragraphs - .escape_colon_emojis_in_codeblock - .unescape_raw_tag_in_codeblocks - .wrap_all_figures_with_tags - .wrap_mentions_with_links - .html - end -end diff --git a/app/services/markdown_processor/parser.rb b/app/services/markdown_processor/parser.rb new file mode 100644 index 000000000..18a393992 --- /dev/null +++ b/app/services/markdown_processor/parser.rb @@ -0,0 +1,147 @@ +module MarkdownProcessor + class Parser + include ApplicationHelper + + BAD_XSS_REGEX = [ + /src=["'](data|&)/i, + %r{data:text/html[,;][\sa-z0-9]*}i, + ].freeze + + WORDS_READ_PER_MINUTE = 275.0 + + def initialize(content, source: nil, user: nil) + @content = content + @source = source + @user = user + end + + def finalize(link_attributes: {}) + options = { hard_wrap: true, filter_html: false, link_attributes: link_attributes } + renderer = Redcarpet::Render::HTMLRouge.new(options) + markdown = Redcarpet::Markdown.new(renderer, Constants::Redcarpet::CONFIG) + catch_xss_attempts(@content) + escaped_content = escape_liquid_tags_in_codeblock(@content) + html = markdown.render(escaped_content) + sanitized_content = sanitize_rendered_markdown(html) + begin + liquid_tag_options = { source: @source, user: @user } + parsed_liquid = Liquid::Template.parse(sanitized_content, liquid_tag_options) + html = markdown.render(parsed_liquid.render) + rescue Liquid::SyntaxError => e + html = e.message + end + + parse_html(html) + end + + def calculate_reading_time + word_count = @content.split(/\W+/).count + (word_count / WORDS_READ_PER_MINUTE).ceil + end + + def evaluate_markdown + return if @content.blank? + + renderer = Redcarpet::Render::HTMLRouge.new(hard_wrap: true, filter_html: false) + markdown = Redcarpet::Markdown.new(renderer, Constants::Redcarpet::CONFIG) + allowed_tags = %w[strong abbr aside em p h1 h2 h3 h4 h5 h6 i u b code pre + br ul ol li small sup sub img a span hr blockquote kbd] + allowed_attributes = %w[href strong em ref rel src title alt class] + ActionController::Base.helpers.sanitize markdown.render(@content), + tags: allowed_tags, + attributes: allowed_attributes + end + + def evaluate_limited_markdown + return if @content.blank? + + renderer = Redcarpet::Render::HTMLRouge.new(hard_wrap: true, filter_html: false) + markdown = Redcarpet::Markdown.new(renderer, Constants::Redcarpet::CONFIG) + allowed_tags = %w[strong i u b em p br code] + allowed_attributes = %w[href strong em ref rel src title alt class] + ActionController::Base.helpers.sanitize markdown.render(@content), + tags: allowed_tags, + attributes: allowed_attributes + end + + def evaluate_inline_limited_markdown + return if @content.blank? + + renderer = Redcarpet::Render::HTMLRouge.new(hard_wrap: true, filter_html: false) + markdown = Redcarpet::Markdown.new(renderer, Constants::Redcarpet::CONFIG) + allowed_tags = %w[strong i u b em code] + allowed_attributes = %w[href strong em ref rel src title alt class] + ActionController::Base.helpers.sanitize markdown.render(@content), + tags: allowed_tags, + attributes: allowed_attributes + end + + def evaluate_listings_markdown + return if @content.blank? + + renderer = Redcarpet::Render::HTMLRouge.new(hard_wrap: true, filter_html: false) + markdown = Redcarpet::Markdown.new(renderer, Constants::Redcarpet::CONFIG) + allowed_tags = %w[strong abbr aside em p h4 h5 h6 i u b code pre + br ul ol li small sup sub a span hr blockquote kbd] + allowed_attributes = %w[href strong em ref rel src title alt class] + ActionController::Base.helpers.sanitize markdown.render(@content), + tags: allowed_tags, + attributes: allowed_attributes + end + + def tags_used + return [] if @content.blank? + + cleaned_parsed = escape_liquid_tags_in_codeblock(@content) + tags = [] + liquid_tag_options = { source: @source, user: @user } + Liquid::Template.parse(cleaned_parsed, liquid_tag_options).root.nodelist.each do |node| + tags << node.class if node.class.superclass.to_s == LiquidTagBase.to_s + end + tags.uniq + rescue Liquid::SyntaxError + [] + end + + def catch_xss_attempts(markdown) + return unless markdown.match?(Regexp.union(BAD_XSS_REGEX)) + + raise ArgumentError, "Invalid markdown detected" + end + + def escape_liquid_tags_in_codeblock(content) + # Escape codeblocks, code spans, and inline code + content.gsub(/[[:space:]]*`{3}.*?`{3}|`{2}.+?`{2}|`{1}.+?`{1}/m) do |codeblock| + codeblock.gsub!("{% endraw %}", "{----% endraw %----}") + codeblock.gsub!("{% raw %}", "{----% raw %----}") + if codeblock.match?(/[[:space:]]*`{3}/) + "\n{% raw %}\n#{codeblock}\n{% endraw %}\n" + else + "{% raw %}#{codeblock}{% endraw %}" + end + end + end + + private + + def parse_html(html) + return html if html.blank? + + Html::Parser + .new(html) + .remove_nested_linebreak_in_list + .prefix_all_images + .wrap_all_images_in_links + .add_control_class_to_codeblock + .add_control_panel_to_codeblock + .add_fullscreen_button_to_panel + .wrap_all_tables + .remove_empty_paragraphs + .escape_colon_emojis_in_codeblock + .unescape_raw_tag_in_codeblocks + .wrap_all_figures_with_tags + .wrap_mentions_with_links + .html + end + end +end diff --git a/spec/services/markdown_parser_spec.rb b/spec/services/markdown_processor/parser_spec.rb similarity index 99% rename from spec/services/markdown_parser_spec.rb rename to spec/services/markdown_processor/parser_spec.rb index c7b714f15..de9aeb858 100644 --- a/spec/services/markdown_parser_spec.rb +++ b/spec/services/markdown_processor/parser_spec.rb @@ -1,6 +1,6 @@ require "rails_helper" -RSpec.describe MarkdownParser, type: :service do +RSpec.describe MarkdownProcessor::Parser, type: :service do let(:random_word) { Faker::Lorem.word } let(:basic_parsed_markdown) { described_class.new(random_word) }