diff --git a/app/controllers/concerns/api/articles_controller.rb b/app/controllers/concerns/api/articles_controller.rb index 5341ab4af..c8ea472e5 100644 --- a/app/controllers/concerns/api/articles_controller.rb +++ b/app/controllers/concerns/api/articles_controller.rb @@ -10,6 +10,11 @@ module Api updated_at video_thumbnail_url reading_time ].freeze + ADDITIONAL_SEARCH_ATTRIBUTES_FOR_SERIALIZATION = [ + *INDEX_ATTRIBUTES_FOR_SERIALIZATION, :body_markdown + ].freeze + private_constant :ADDITIONAL_SEARCH_ATTRIBUTES_FOR_SERIALIZATION + SHOW_ATTRIBUTES_FOR_SERIALIZATION = [ *INDEX_ATTRIBUTES_FOR_SERIALIZATION, :body_markdown, :processed_html ].freeze @@ -118,6 +123,21 @@ module Api end end + def search + # I temporarily added a new search endpoint in the interest of getting the chatGPT plugin live without changing + # the existing index endpoint. There are some experiments which we want to conduct which I think makes sense on + # a new endpoint rather than an existing one. We may want to refactor the index one in the future. + @articles = Articles::ApiSearchQuery.call(params) + + # This adds some inconsistency where we omit the body markdown when the response has more than 1 article because + # ChatGPT cannot process the long body request. + @articles = if @articles.count > 1 + @articles.select(INDEX_ATTRIBUTES_FOR_SERIALIZATION).decorate + else + @articles.select(ADDITIONAL_SEARCH_ATTRIBUTES_FOR_SERIALIZATION).decorate + end + end + private def per_page_max diff --git a/app/queries/articles/api_search_query.rb b/app/queries/articles/api_search_query.rb new file mode 100644 index 000000000..9d3390c70 --- /dev/null +++ b/app/queries/articles/api_search_query.rb @@ -0,0 +1,50 @@ +module Articles + class ApiSearchQuery + DEFAULT_PER_PAGE = 30 + + def self.call(...) + new(...).call + end + + def initialize(params) + @q = params[:q] + @top = params[:top] + @page = params[:page].to_i + @per_page = [(params[:per_page] || DEFAULT_PER_PAGE).to_i, per_page_max].min + end + + def call + @articles = published_articles_with_users_and_organizations + + if q.present? + @articles = query_articles + end + + if top.present? + @articles = top_articles.order(public_reactions_count: :desc) + end + + @articles.page(page).per(per_page || DEFAULT_PER_PAGE) + end + + private + + attr_reader :q, :top, :page, :per_page + + def per_page_max + (ApplicationConfig["API_PER_PAGE_MAX"] || 1000).to_i + end + + def query_articles + @articles.search_articles(q) + end + + def top_articles + @articles.where("published_at > ?", top.to_i.days.ago) + end + + def published_articles_with_users_and_organizations + Article.published.includes([{ user: :profile }, :organization]).order(hotness_score: :desc) + end + end +end diff --git a/app/views/api/v0/articles/search.json.jbuilder b/app/views/api/v0/articles/search.json.jbuilder new file mode 100644 index 000000000..23cc776b6 --- /dev/null +++ b/app/views/api/v0/articles/search.json.jbuilder @@ -0,0 +1,23 @@ +json.array! @articles do |article| + json.partial! "api/v0/articles/article", article: article + + json.body_markdown article.body_markdown if article.respond_to?(:body_markdown) + + # /api/articles and /api/articles/:id have opposite representations + # of `tag_list` and `tags and we can't align them without breaking the API, + # this is fully documented in the API docs + # see for more details + json.tag_list article.cached_tag_list_array + json.tags article.cached_tag_list + + json.partial! "api/v0/shared/user", user: article.user + + if article.organization + json.partial! "api/v0/shared/organization", organization: article.organization + end + + flare_tag = FlareTag.new(article).tag + if flare_tag + json.partial! "api/v0/articles/flare_tag", flare_tag: flare_tag + end +end diff --git a/app/views/api/v1/articles/search.json.jbuilder b/app/views/api/v1/articles/search.json.jbuilder new file mode 100644 index 000000000..285917f29 --- /dev/null +++ b/app/views/api/v1/articles/search.json.jbuilder @@ -0,0 +1,23 @@ +json.array! @articles do |article| + json.partial! "api/v1/articles/article", article: article + + json.body_markdown article.body_markdown if article.respond_to?(:body_markdown) + + # /api/articles and /api/articles/:id have opposite representations + # of `tag_list` and `tags and we can't align them without breaking the API, + # this is fully documented in the API docs + # see for more details + json.tag_list article.cached_tag_list_array + json.tags article.cached_tag_list + + json.partial! "api/v1/shared/user", user: article.user + + if article.organization + json.partial! "api/v1/shared/organization", organization: article.organization + end + + flare_tag = FlareTag.new(article).tag + if flare_tag + json.partial! "api/v1/articles/flare_tag", flare_tag: flare_tag + end +end diff --git a/config/initializers/cors.rb b/config/initializers/cors.rb index d41b7da16..55b94f811 100644 --- a/config/initializers/cors.rb +++ b/config/initializers/cors.rb @@ -17,6 +17,12 @@ Rails.application.config.middleware.insert_before( source # echo back the client's `Origin` header instead of using `*` end + resource "/.well-known/ai-plugin.json" + resource "/openapi.yml" + resource "/api/articles/search/*", methods: %i[head get options], headers: %w[content-type openai-conversation-id + openai-ephemeral-user-id], + max_age: 2.hours.to_i + # allowed public APIs %w[articles comments listings podcast_episodes tags users videos].each do |resource_name| # allow read operations, disallow custom headers (eg. api-key) and enable preflight caching diff --git a/config/routes/api.rb b/config/routes/api.rb index dd9e74bd6..8f7de7642 100644 --- a/config/routes/api.rb +++ b/config/routes/api.rb @@ -4,11 +4,13 @@ end resources :articles, only: %i[index show create update] do collection do + get "/search", to: "articles#search" get "me(/:status)", to: "articles#me", constraints: { status: /published|unpublished|all/ } get "/:username/:slug", to: "articles#show_by_slug" get "/latest", to: "articles#index", defaults: { sort: "desc" } end end + resources :comments, only: %i[index show] resources :videos, only: [:index] resources :podcast_episodes, only: [:index] diff --git a/public/.well-known/ai-plugin.json b/public/.well-known/ai-plugin.json new file mode 100644 index 000000000..dba47fad6 --- /dev/null +++ b/public/.well-known/ai-plugin.json @@ -0,0 +1,18 @@ +{ + "schema_version": "v1", + "name_for_human": "DEV Community", + "name_for_model": "dev", + "description_for_human": "Plugin for recommending articles or users from DEV Community.", + "description_for_model": "Plugin for recommending articles or users from DEV Community. Always link to a url for the resource returned.", + "auth": { + "type": "none" + }, + "api": { + "type": "openapi", + "url": "https://dev.to/openapi.yml", + "is_user_authenticated": false + }, + "logo_url": "https://dev.to/logo.png", + "contact_email": "yo@dev.to", + "legal_info_url": "https://dev.to/terms" +} diff --git a/public/logo.png b/public/logo.png new file mode 100644 index 000000000..436117abf Binary files /dev/null and b/public/logo.png differ diff --git a/public/openapi.yml b/public/openapi.yml new file mode 100644 index 000000000..db27fa569 --- /dev/null +++ b/public/openapi.yml @@ -0,0 +1,112 @@ +openapi: 3.0.1 +# We start by defining the specification version, the title, description, and version number. When a query is run in ChatGPT, it will look at the description that is defined in the info section to determine if the plugin is relevant for the user query. +info: + title: DEV Community + description: A plugin that recommends resources like articles or users to a user using ChatGP. + version: 'v1' +servers: + - url: https://dev.to +paths: + /api/articles/search: + get: + operationId: getArticles + summary: Get a list of filtered articles + parameters: + - in: "query" + name: "q" + required: false + description: "Accepts keywords to use as a search query." + schema: + type: "string" + - in: "query" + name: "page" + required: false + description: "Pagination Page" + schema: + type: "integer" + format: "int32" + minimum: 0 + default: 0 + - in: "query" + name: "per_page" + required: false + description: "Page size (the number of items to return per page)." + schema: + type: "integer" + format: "int32" + minimum: 1 + maximum: 100 + default: 60 + - in: "query" + name: "top" + required: false + description: "Returns the most popular articles in the last N days. 'top' indicates the number of days since publication of the articles returned. This param can be used in conjuction with q or tag." + schema: + type: "string" + responses: + "200": + description: OK + content: + application/vnd.forem.api-v1+json: + schema: + $ref: '#/components/schemas/getArticlesResponse' +components: + schemas: + getArticlesResponse: + description: "Representation of an article returned in a list" + type: "object" + properties: + type_of: { type: "string" } + id: { type: "integer", format: "int32" } + title: { type: "string", description: "The article title" } + description: { type: "string", description: "A description of the article" } + cover_image: { type: "string", format: "url", nullable: true } + readable_publish_date: { type: "string" } + social_image: { type: "string", format: "url" } + tag_list: + type: "array" + description: "An array representation of the tags that are associated with an article" + items: + type: "string" + tags: { type: "string", description: "An array representation of the tags that are associated with an article" } + slug: { type: "string" } + path: { description: "A relative path of the article.", type: "string", format: "path" } + url: { type: "string", format: "url", description: "The url of the article. Can be used to link to the article." } + body_markdown: {type: "string", description: "The body of the article" } + canonical_url: { type: "string", format: "url" } + positive_reactions_count: { type: "integer", format: "int32" } + public_reactions_count: { type: "integer", format: "int32" } + created_at: { type: "string", format: "date-time" } + edited_at: { type: "string", format: "date-time", nullable: true } + crossposted_at: { type: "string", format: "date-time", nullable: true } + published_at: { type: "string", format: "date-time" } + last_comment_at: { type: "string", format: "date-time" } + published_timestamp: { description: "Crossposting or published date time", type: "string", + format: "date-time" } + reading_time_minutes: { description: "Reading time, in minutes", type: "integer", format: "int32" } + user: + $ref: "#/components/schemas/SharedUser" + organization: + $ref: "#/components/schemas/SharedOrganization" + required: ["type_of", "id", "title", "description", "cover_image", "readable_publish_date", "social_image", "tag_list", "tags", "slug", "path", "url", "canonical_url", "comments_count", "positive_reactions_count", "public_reactions_count", "created_at", "edited_at", "crossposted_at", "published_at", "last_comment_at", "published_timestamp", "user", "reading_time_minutes"] + SharedUser: + description: "The author" + type: "object" + properties: + name: { type: "string" } + username: { type: "string" } + twitter_username: { type: "string", nullable: true } + github_username: { type: "string", nullable: true } + website_url: { type: "string", format: :url, nullable: true } + profile_image: { description: "Profile image (640x640)", type: "string" } + profile_image_90: { description: "Profile image (90x90)", type: "string" } + SharedOrganization: + description: "The organization the resource belongs to" + type: "object" + properties: + name: { type: "string" } + username: { type: "string" } + slug: { type: "string" } + profile_image: { description: "Profile image (640x640)", type: "string", format: :url } + profile_image_90: { description: "Profile image (90x90)", type: "string", format: :url } + diff --git a/spec/queries/articles/api_search_query_spec.rb b/spec/queries/articles/api_search_query_spec.rb new file mode 100644 index 000000000..118bbc79c --- /dev/null +++ b/spec/queries/articles/api_search_query_spec.rb @@ -0,0 +1,37 @@ +require "rails_helper" + +RSpec.describe Articles::ApiSearchQuery, type: :query do + before do + create(:article, published: false) + create(:article, title: "Top ten Interview tips") + create(:article, title: "Top ten Ruby tips") + create(:article, title: "Frontend Frameworks") + create(:article) + end + + context "when there is no query parameter" do + it "shows all published and approved articles" do + articles = described_class.call({}) + # The one not included has publiched set to false. + expect(articles.count).to eq(4) + end + end + + context "when there is a query parameter" do + it "shows articles that match that query" do + articles = described_class.call({ q: "ruby" }) + expect(articles.count).to eq(1) + end + end + + context "when there is a top parameter" do + it "shows the most popular articles in the last n days" do + article = Article.find_by(title: "Frontend Frameworks") + article.update_column(:published_at, 30.days.ago) + + # The two not included is the one that has publiched set to false. + articles = described_class.call({ top: 10 }) + expect(articles.count).to eq(3) + end + end +end diff --git a/spec/requests/api/v0/articles_spec.rb b/spec/requests/api/v0/articles_spec.rb index 13c0f47e6..ceb47f418 100644 --- a/spec/requests/api/v0/articles_spec.rb +++ b/spec/requests/api/v0/articles_spec.rb @@ -1174,4 +1174,89 @@ RSpec.describe "Api::V0::Articles" do end end end + + describe "GET /api/articles/search" do + before { article } + + it "returns CORS headers" do + origin = "http://example.com" + get "/api/articles/search", headers: { origin: origin } + + expect(response).to have_http_status(:ok) + expect(response.headers["Access-Control-Allow-Origin"]).to eq(origin) + expect(response.headers["Access-Control-Allow-Methods"]).to eq("HEAD, GET, OPTIONS") + expect(response.headers["Access-Control-Expose-Headers"]).to be_empty + expect(response.headers["Access-Control-Max-Age"]).to eq(2.hours.to_i.to_s) + end + + context "when there is one article returned" do + it "has correct keys in the response" do + article.update_columns(organization_id: organization.id) + get "/api/articles/search" + + index_keys = %w[ + type_of id title description cover_image readable_publish_date social_image + tag_list tags slug path url canonical_url comments_count public_reactions_count positive_reactions_count + collection_id created_at edited_at crossposted_at published_at last_comment_at + published_timestamp user organization flare_tag reading_time_minutes body_markdown + ] + + expect(response.parsed_body.first.keys).to match_array index_keys + end + end + + context "when there is more than one article returned" do + it "has correct keys in the response" do + new_article = create(:article) + article.update_columns(organization_id: organization.id) + new_article.update_columns(organization_id: organization.id) + + get "/api/articles/search" + + keys = %w[ + type_of id title description cover_image readable_publish_date social_image + tag_list tags slug path url canonical_url comments_count public_reactions_count positive_reactions_count + collection_id created_at edited_at crossposted_at published_at last_comment_at + published_timestamp user organization flare_tag reading_time_minutes + ] + + expect(response.parsed_body.first.keys).to match_array keys + end + end + + it "supports pagination" do + create_list(:article, 2) + get "/api/articles/search", params: { page: 1, per_page: 2 } + expect(response.parsed_body.length).to eq(2) + get "/api/articles/search", params: { page: 2, per_page: 2 } + expect(response.parsed_body.length).to eq(1) + end + + it "returns flare tag in the response" do + get "/api/articles/search" + response_article = response.parsed_body.first + expect(response_article["flare_tag"]).to be_present + expect(response_article["flare_tag"].keys).to eq(%w[name bg_color_hex text_color_hex]) + expect(response_article["flare_tag"]["name"]).to eq("discuss") + end + + context "with regression tests" do + it "works if both the social image and the main image are missing" do + article.update_columns(social_image: nil, main_image: nil) + + get "/api/articles/search" + expect(response).to have_http_status(:ok) + end + + it "respects API_PER_PAGE_MAX limit set in ENV variable" do + allow(ApplicationConfig).to receive(:[]).and_return(nil) + allow(ApplicationConfig).to receive(:[]).with("APP_PROTOCOL").and_return("http://") + allow(ApplicationConfig).to receive(:[]).with("API_PER_PAGE_MAX").and_return(2) + + create_list(:article, 3, tags: "discuss", public_reactions_count: 1, score: 1, published: true, featured: true) + get "/api/articles/search", params: { per_page: 10 } + expect(response.parsed_body.count).to eq(2) + end + end + end end diff --git a/spec/requests/api/v1/articles_spec.rb b/spec/requests/api/v1/articles_spec.rb index bce81450e..a3bebf29b 100644 --- a/spec/requests/api/v1/articles_spec.rb +++ b/spec/requests/api/v1/articles_spec.rb @@ -1226,4 +1226,89 @@ RSpec.describe "Api::V1::Articles" do end end end + + describe "GET /api/articles/search" do + before { article } + + it "returns CORS headers" do + origin = "http://example.com" + get "/api/articles/search", headers: { origin: origin } + + expect(response).to have_http_status(:ok) + expect(response.headers["Access-Control-Allow-Origin"]).to eq(origin) + expect(response.headers["Access-Control-Allow-Methods"]).to eq("HEAD, GET, OPTIONS") + expect(response.headers["Access-Control-Expose-Headers"]).to be_empty + expect(response.headers["Access-Control-Max-Age"]).to eq(2.hours.to_i.to_s) + end + + context "when there is one article returned" do + it "has correct keys in the response" do + article.update_columns(organization_id: organization.id) + get "/api/articles/search" + + index_keys = %w[ + type_of id title description cover_image readable_publish_date social_image + tag_list tags slug path url canonical_url comments_count public_reactions_count positive_reactions_count + collection_id created_at edited_at crossposted_at published_at last_comment_at + published_timestamp user organization flare_tag reading_time_minutes body_markdown + ] + + expect(response.parsed_body.first.keys).to match_array index_keys + end + end + + context "when there is more than one article returned" do + it "has correct keys in the response" do + new_article = create(:article) + article.update_columns(organization_id: organization.id) + new_article.update_columns(organization_id: organization.id) + + get "/api/articles/search" + + keys = %w[ + type_of id title description cover_image readable_publish_date social_image + tag_list tags slug path url canonical_url comments_count public_reactions_count positive_reactions_count + collection_id created_at edited_at crossposted_at published_at last_comment_at + published_timestamp user organization flare_tag reading_time_minutes + ] + + expect(response.parsed_body.first.keys).to match_array keys + end + end + + it "supports pagination" do + create_list(:article, 2) + get "/api/articles/search", params: { page: 1, per_page: 2 } + expect(response.parsed_body.length).to eq(2) + get "/api/articles/search", params: { page: 2, per_page: 2 } + expect(response.parsed_body.length).to eq(1) + end + + it "returns flare tag in the response" do + get "/api/articles/search" + response_article = response.parsed_body.first + expect(response_article["flare_tag"]).to be_present + expect(response_article["flare_tag"].keys).to eq(%w[name bg_color_hex text_color_hex]) + expect(response_article["flare_tag"]["name"]).to eq("discuss") + end + + context "with regression tests" do + it "works if both the social image and the main image are missing" do + article.update_columns(social_image: nil, main_image: nil) + + get "/api/articles/search" + expect(response).to have_http_status(:ok) + end + + it "respects API_PER_PAGE_MAX limit set in ENV variable" do + allow(ApplicationConfig).to receive(:[]).and_return(nil) + allow(ApplicationConfig).to receive(:[]).with("APP_PROTOCOL").and_return("http://") + allow(ApplicationConfig).to receive(:[]).with("API_PER_PAGE_MAX").and_return(2) + + create_list(:article, 3, tags: "discuss", public_reactions_count: 1, score: 1, published: true, featured: true) + get "/api/articles/search", params: { per_page: 10 } + expect(response.parsed_body.count).to eq(2) + end + end + end end