Preparation of endpoint and files required for ChatGPT Plugin (#19394)

* feat: add the ./wellknown/ai-plugin.json file

* feat: add a logo

* feat: firts attempt at open_api file with endpoint we thought we'd use

* feat: add a search action to teh articles controller that build up teh appropriate json

* feat: add a query param to the article index

* feat: add some chatgpt endpoints and file paths to CORS

* feat: update the openapi yml file to reflect the article search endpoint

* fix: use the correct concept

* feat: replace the logo

* feat: update the ai-json file

* feat: update the openapi file

* feat: update the api and it's params

* feat: update the search query

* fix: ordering

* feat: cors debug

* fix: logic with markdown

* chore: fix test

* spec: article

* spec: article

* spec: article

* Update public/.well-known/ai-plugin.json

* Update public/openapi.yml

---------

Co-authored-by: Ben Halpern <bendhalpern@gmail.com>

app/controllers/concerns/api/articles_controller.rb

@@ -10,6 +10,11 @@ module Api
       updated_at video_thumbnail_url reading_time
     ].freeze
 
+    ADDITIONAL_SEARCH_ATTRIBUTES_FOR_SERIALIZATION = [
+      *INDEX_ATTRIBUTES_FOR_SERIALIZATION, :body_markdown
+    ].freeze
+    private_constant :ADDITIONAL_SEARCH_ATTRIBUTES_FOR_SERIALIZATION
+
     SHOW_ATTRIBUTES_FOR_SERIALIZATION = [
       *INDEX_ATTRIBUTES_FOR_SERIALIZATION, :body_markdown, :processed_html
     ].freeze
@@ -118,6 +123,21 @@ module Api
       end
     end
 
+    def search
+      # I temporarily added a new search endpoint in the interest of getting the chatGPT plugin live without changing
+      # the existing index endpoint. There are some experiments which we want to conduct which I think makes sense on
+      # a new endpoint rather than an existing one. We may want to refactor the index one in the future.
+      @articles = Articles::ApiSearchQuery.call(params)
+
+      # This adds some inconsistency where we omit the body markdown when the response has more than 1 article because
+      # ChatGPT cannot process the long body request.
+      @articles = if @articles.count > 1
+                    @articles.select(INDEX_ATTRIBUTES_FOR_SERIALIZATION).decorate
+                  else
+                    @articles.select(ADDITIONAL_SEARCH_ATTRIBUTES_FOR_SERIALIZATION).decorate
+                  end
+    end
+
     private
 
     def per_page_max

app/queries/articles/api_search_query.rb

@@ -0,0 +1,50 @@
+module Articles
+  class ApiSearchQuery
+    DEFAULT_PER_PAGE = 30
+
+    def self.call(...)
+      new(...).call
+    end
+
+    def initialize(params)
+      @q = params[:q]
+      @top = params[:top]
+      @page = params[:page].to_i
+      @per_page = [(params[:per_page] || DEFAULT_PER_PAGE).to_i, per_page_max].min
+    end
+
+    def call
+      @articles = published_articles_with_users_and_organizations
+
+      if q.present?
+        @articles = query_articles
+      end
+
+      if top.present?
+        @articles = top_articles.order(public_reactions_count: :desc)
+      end
+
+      @articles.page(page).per(per_page || DEFAULT_PER_PAGE)
+    end
+
+    private
+
+    attr_reader :q, :top, :page, :per_page
+
+    def per_page_max
+      (ApplicationConfig["API_PER_PAGE_MAX"] || 1000).to_i
+    end
+
+    def query_articles
+      @articles.search_articles(q)
+    end
+
+    def top_articles
+      @articles.where("published_at > ?", top.to_i.days.ago)
+    end
+
+    def published_articles_with_users_and_organizations
+      Article.published.includes([{ user: :profile }, :organization]).order(hotness_score: :desc)
+    end
+  end
+end

app/views/api/v0/articles/search.json.jbuilder

@@ -0,0 +1,23 @@
+json.array! @articles do |article|
+  json.partial! "api/v0/articles/article", article: article
+
+  json.body_markdown article.body_markdown if article.respond_to?(:body_markdown)
+
+  # /api/articles and /api/articles/:id have opposite representations
+  # of `tag_list` and `tags and we can't align them without breaking the API,
+  # this is fully documented in the API docs
+  # see <https://github.com/forem/forem/issues/4206> for more details
+  json.tag_list article.cached_tag_list_array
+  json.tags article.cached_tag_list
+
+  json.partial! "api/v0/shared/user", user: article.user
+
+  if article.organization
+    json.partial! "api/v0/shared/organization", organization: article.organization
+  end
+
+  flare_tag = FlareTag.new(article).tag
+  if flare_tag
+    json.partial! "api/v0/articles/flare_tag", flare_tag: flare_tag
+  end
+end

app/views/api/v1/articles/search.json.jbuilder

@@ -0,0 +1,23 @@
+json.array! @articles do |article|
+  json.partial! "api/v1/articles/article", article: article
+
+  json.body_markdown article.body_markdown if article.respond_to?(:body_markdown)
+
+  # /api/articles and /api/articles/:id have opposite representations
+  # of `tag_list` and `tags and we can't align them without breaking the API,
+  # this is fully documented in the API docs
+  # see <https://github.com/forem/forem/issues/4206> for more details
+  json.tag_list article.cached_tag_list_array
+  json.tags article.cached_tag_list
+
+  json.partial! "api/v1/shared/user", user: article.user
+
+  if article.organization
+    json.partial! "api/v1/shared/organization", organization: article.organization
+  end
+
+  flare_tag = FlareTag.new(article).tag
+  if flare_tag
+    json.partial! "api/v1/articles/flare_tag", flare_tag: flare_tag
+  end
+end

config/initializers/cors.rb

@@ -17,6 +17,12 @@ Rails.application.config.middleware.insert_before(
       source # echo back the client's `Origin` header instead of using `*`
     end
 
+    resource "/.well-known/ai-plugin.json"
+    resource "/openapi.yml"
+    resource "/api/articles/search/*", methods: %i[head get options], headers: %w[content-type openai-conversation-id
+                                                                                  openai-ephemeral-user-id],
+                                       max_age: 2.hours.to_i
+
     # allowed public APIs
     %w[articles comments listings podcast_episodes tags users videos].each do |resource_name|
       # allow read operations, disallow custom headers (eg. api-key) and enable preflight caching

config/routes/api.rb

@@ -4,11 +4,13 @@ end
 
 resources :articles, only: %i[index show create update] do
   collection do
+    get "/search", to: "articles#search"
     get "me(/:status)", to: "articles#me", constraints: { status: /published|unpublished|all/ }
     get "/:username/:slug", to: "articles#show_by_slug"
     get "/latest", to: "articles#index", defaults: { sort: "desc" }
   end
 end
+
 resources :comments, only: %i[index show]
 resources :videos, only: [:index]
 resources :podcast_episodes, only: [:index]

public/.well-known/ai-plugin.json

@@ -0,0 +1,18 @@
+{
+  "schema_version": "v1",
+  "name_for_human": "DEV Community",
+  "name_for_model": "dev",
+  "description_for_human": "Plugin for recommending articles or users from DEV Community.",
+  "description_for_model": "Plugin for recommending articles or users from DEV Community. Always link to a url for the resource returned.",
+  "auth": {
+    "type": "none"
+  },
+  "api": {
+    "type": "openapi",
+    "url": "https://dev.to/openapi.yml",
+    "is_user_authenticated": false
+  },
+  "logo_url": "https://dev.to/logo.png",
+  "contact_email": "yo@dev.to",
+  "legal_info_url": "https://dev.to/terms"
+}

public/openapi.yml

@@ -0,0 +1,112 @@
+openapi: 3.0.1
+# We start by defining the specification version, the title, description, and version number. When a query is run in ChatGPT, it will look at the description that is defined in the info section to determine if the plugin is relevant for the user query.
+info:
+  title: DEV Community
+  description: A plugin that recommends resources like articles or users to a user using ChatGP.
+  version: 'v1'
+servers:
+  - url: https://dev.to
+paths:
+  /api/articles/search:
+    get:
+      operationId: getArticles
+      summary: Get a list of filtered articles
+      parameters:
+        - in: "query"
+          name: "q"
+          required: false
+          description: "Accepts keywords to use as a search query."
+          schema:
+            type: "string"
+        - in: "query"
+          name: "page"
+          required: false
+          description: "Pagination Page"
+          schema:
+            type: "integer"
+            format: "int32"
+            minimum: 0
+            default: 0
+        - in: "query"
+          name: "per_page"
+          required: false
+          description: "Page size (the number of items to return per page)."
+          schema:
+            type: "integer"
+            format: "int32"
+            minimum: 1
+            maximum: 100
+            default: 60
+        - in: "query"
+          name: "top"
+          required: false
+          description: "Returns the most popular articles in the last N days. 'top' indicates the number of days since publication of the articles returned. This param can be used in conjuction with q or tag."
+          schema:
+            type: "string"
+      responses:
+        "200":
+          description: OK
+          content:
+            application/vnd.forem.api-v1+json:
+              schema:
+                $ref: '#/components/schemas/getArticlesResponse'
+components:
+  schemas:
+    getArticlesResponse:
+      description: "Representation of an article returned in a list"
+      type: "object"
+      properties:
+        type_of: { type: "string" }
+        id: { type: "integer", format: "int32" }
+        title: { type: "string", description: "The article title" }
+        description: { type: "string", description: "A description of the article" }
+        cover_image: { type: "string", format: "url", nullable: true }
+        readable_publish_date: { type: "string" }
+        social_image: { type: "string", format: "url" }
+        tag_list:
+          type: "array"
+          description: "An array representation of the tags that are associated with an article"
+          items:
+            type: "string"
+        tags: { type: "string", description: "An array representation of the tags that are associated with an article" }
+        slug: { type: "string" }
+        path: { description: "A relative path of the article.", type: "string", format: "path" }
+        url: { type: "string", format: "url", description: "The url of the article. Can be used to link to the article." }
+        body_markdown: {type: "string", description: "The body of the article" }
+        canonical_url: { type: "string", format: "url" }
+        positive_reactions_count: { type: "integer", format: "int32" }
+        public_reactions_count: { type: "integer", format: "int32" }
+        created_at: { type: "string", format: "date-time" }
+        edited_at: { type: "string", format: "date-time", nullable: true }
+        crossposted_at: { type: "string", format: "date-time", nullable: true }
+        published_at: { type: "string", format: "date-time" }
+        last_comment_at: { type: "string", format: "date-time" }
+        published_timestamp: { description: "Crossposting or published date time", type: "string",
+                                format: "date-time" }
+        reading_time_minutes: { description: "Reading time, in minutes", type: "integer", format: "int32" }
+        user:
+          $ref: "#/components/schemas/SharedUser"
+        organization:
+          $ref: "#/components/schemas/SharedOrganization"
+      required: ["type_of", "id", "title", "description", "cover_image", "readable_publish_date", "social_image", "tag_list", "tags", "slug", "path", "url", "canonical_url", "comments_count", "positive_reactions_count", "public_reactions_count", "created_at", "edited_at", "crossposted_at", "published_at", "last_comment_at", "published_timestamp", "user", "reading_time_minutes"]
+    SharedUser:
+      description: "The author"
+      type: "object"
+      properties:
+        name: { type: "string" }
+        username: { type: "string" }
+        twitter_username: { type: "string", nullable: true }
+        github_username: { type: "string", nullable: true }
+        website_url: { type: "string", format: :url, nullable: true }
+        profile_image: { description: "Profile image (640x640)", type: "string" }
+        profile_image_90: { description: "Profile image (90x90)", type: "string" }
+    SharedOrganization:
+      description: "The organization the resource belongs to"
+      type: "object"
+      properties:
+        name: { type: "string" }
+        username: { type: "string" }
+        slug: { type: "string" }
+        profile_image: { description: "Profile image (640x640)", type: "string", format: :url }
+        profile_image_90: { description: "Profile image (90x90)", type: "string", format: :url }
+

spec/queries/articles/api_search_query_spec.rb

@@ -0,0 +1,37 @@
+require "rails_helper"
+
+RSpec.describe Articles::ApiSearchQuery, type: :query do
+  before do
+    create(:article, published: false)
+    create(:article, title: "Top ten Interview tips")
+    create(:article, title: "Top ten Ruby tips")
+    create(:article, title: "Frontend Frameworks")
+    create(:article)
+  end
+
+  context "when there is no query parameter" do
+    it "shows all published and approved articles" do
+      articles = described_class.call({})
+      # The one not included has publiched set to false.
+      expect(articles.count).to eq(4)
+    end
+  end
+
+  context "when there is a query parameter" do
+    it "shows articles that match that query" do
+      articles = described_class.call({ q: "ruby" })
+      expect(articles.count).to eq(1)
+    end
+  end
+
+  context "when there is a top parameter" do
+    it "shows the most popular articles in the last n days" do
+      article = Article.find_by(title: "Frontend Frameworks")
+      article.update_column(:published_at, 30.days.ago)
+
+      # The two not included is the one that has publiched set to false.
+      articles = described_class.call({ top: 10 })
+      expect(articles.count).to eq(3)
+    end
+  end
+end

spec/requests/api/v0/articles_spec.rb

@@ -1174,4 +1174,89 @@ RSpec.describe "Api::V0::Articles" do
       end
     end
   end
+
+  describe "GET /api/articles/search" do
+    before { article }
+
+    it "returns CORS headers" do
+      origin = "http://example.com"
+      get "/api/articles/search", headers: { origin: origin }
+
+      expect(response).to have_http_status(:ok)
+      expect(response.headers["Access-Control-Allow-Origin"]).to eq(origin)
+      expect(response.headers["Access-Control-Allow-Methods"]).to eq("HEAD, GET, OPTIONS")
+      expect(response.headers["Access-Control-Expose-Headers"]).to be_empty
+      expect(response.headers["Access-Control-Max-Age"]).to eq(2.hours.to_i.to_s)
+    end
+
+    context "when there is one article returned" do
+      it "has correct keys in the response" do
+        article.update_columns(organization_id: organization.id)
+        get "/api/articles/search"
+
+        index_keys = %w[
+          type_of id title description cover_image readable_publish_date social_image
+          tag_list tags slug path url canonical_url comments_count public_reactions_count positive_reactions_count
+          collection_id created_at edited_at crossposted_at published_at last_comment_at
+          published_timestamp user organization flare_tag reading_time_minutes body_markdown
+        ]
+
+        expect(response.parsed_body.first.keys).to match_array index_keys
+      end
+    end
+
+    context "when there is more than one article returned" do
+      it "has correct keys in the response" do
+        new_article = create(:article)
+        article.update_columns(organization_id: organization.id)
+        new_article.update_columns(organization_id: organization.id)
+
+        get "/api/articles/search"
+
+        keys = %w[
+          type_of id title description cover_image readable_publish_date social_image
+          tag_list tags slug path url canonical_url comments_count public_reactions_count positive_reactions_count
+          collection_id created_at edited_at crossposted_at published_at last_comment_at
+          published_timestamp user organization flare_tag reading_time_minutes
+        ]
+
+        expect(response.parsed_body.first.keys).to match_array keys
+      end
+    end
+
+    it "supports pagination" do
+      create_list(:article, 2)
+      get "/api/articles/search", params: { page: 1, per_page: 2 }
+      expect(response.parsed_body.length).to eq(2)
+      get "/api/articles/search", params: { page: 2, per_page: 2 }
+      expect(response.parsed_body.length).to eq(1)
+    end
+
+    it "returns flare tag in the response" do
+      get "/api/articles/search"
+      response_article = response.parsed_body.first
+      expect(response_article["flare_tag"]).to be_present
+      expect(response_article["flare_tag"].keys).to eq(%w[name bg_color_hex text_color_hex])
+      expect(response_article["flare_tag"]["name"]).to eq("discuss")
+    end
+
+    context "with regression tests" do
+      it "works if both the social image and the main image are missing" do
+        article.update_columns(social_image: nil, main_image: nil)
+
+        get "/api/articles/search"
+        expect(response).to have_http_status(:ok)
+      end
+
+      it "respects API_PER_PAGE_MAX limit set in ENV variable" do
+        allow(ApplicationConfig).to receive(:[]).and_return(nil)
+        allow(ApplicationConfig).to receive(:[]).with("APP_PROTOCOL").and_return("http://")
+        allow(ApplicationConfig).to receive(:[]).with("API_PER_PAGE_MAX").and_return(2)
+
+        create_list(:article, 3, tags: "discuss", public_reactions_count: 1, score: 1, published: true, featured: true)
+        get "/api/articles/search", params: { per_page: 10 }
+        expect(response.parsed_body.count).to eq(2)
+      end
+    end
+  end
 end

spec/requests/api/v1/articles_spec.rb

@@ -1226,4 +1226,89 @@ RSpec.describe "Api::V1::Articles" do
       end
     end
   end
+
+  describe "GET /api/articles/search" do
+    before { article }
+
+    it "returns CORS headers" do
+      origin = "http://example.com"
+      get "/api/articles/search", headers: { origin: origin }
+
+      expect(response).to have_http_status(:ok)
+      expect(response.headers["Access-Control-Allow-Origin"]).to eq(origin)
+      expect(response.headers["Access-Control-Allow-Methods"]).to eq("HEAD, GET, OPTIONS")
+      expect(response.headers["Access-Control-Expose-Headers"]).to be_empty
+      expect(response.headers["Access-Control-Max-Age"]).to eq(2.hours.to_i.to_s)
+    end
+
+    context "when there is one article returned" do
+      it "has correct keys in the response" do
+        article.update_columns(organization_id: organization.id)
+        get "/api/articles/search"
+
+        index_keys = %w[
+          type_of id title description cover_image readable_publish_date social_image
+          tag_list tags slug path url canonical_url comments_count public_reactions_count positive_reactions_count
+          collection_id created_at edited_at crossposted_at published_at last_comment_at
+          published_timestamp user organization flare_tag reading_time_minutes body_markdown
+        ]
+
+        expect(response.parsed_body.first.keys).to match_array index_keys
+      end
+    end
+
+    context "when there is more than one article returned" do
+      it "has correct keys in the response" do
+        new_article = create(:article)
+        article.update_columns(organization_id: organization.id)
+        new_article.update_columns(organization_id: organization.id)
+
+        get "/api/articles/search"
+
+        keys = %w[
+          type_of id title description cover_image readable_publish_date social_image
+          tag_list tags slug path url canonical_url comments_count public_reactions_count positive_reactions_count
+          collection_id created_at edited_at crossposted_at published_at last_comment_at
+          published_timestamp user organization flare_tag reading_time_minutes
+        ]
+
+        expect(response.parsed_body.first.keys).to match_array keys
+      end
+    end
+
+    it "supports pagination" do
+      create_list(:article, 2)
+      get "/api/articles/search", params: { page: 1, per_page: 2 }
+      expect(response.parsed_body.length).to eq(2)
+      get "/api/articles/search", params: { page: 2, per_page: 2 }
+      expect(response.parsed_body.length).to eq(1)
+    end
+
+    it "returns flare tag in the response" do
+      get "/api/articles/search"
+      response_article = response.parsed_body.first
+      expect(response_article["flare_tag"]).to be_present
+      expect(response_article["flare_tag"].keys).to eq(%w[name bg_color_hex text_color_hex])
+      expect(response_article["flare_tag"]["name"]).to eq("discuss")
+    end
+
+    context "with regression tests" do
+      it "works if both the social image and the main image are missing" do
+        article.update_columns(social_image: nil, main_image: nil)
+
+        get "/api/articles/search"
+        expect(response).to have_http_status(:ok)
+      end
+
+      it "respects API_PER_PAGE_MAX limit set in ENV variable" do
+        allow(ApplicationConfig).to receive(:[]).and_return(nil)
+        allow(ApplicationConfig).to receive(:[]).with("APP_PROTOCOL").and_return("http://")
+        allow(ApplicationConfig).to receive(:[]).with("API_PER_PAGE_MAX").and_return(2)
+
+        create_list(:article, 3, tags: "discuss", public_reactions_count: 1, score: 1, published: true, featured: true)
+        get "/api/articles/search", params: { per_page: 10 }
+        expect(response.parsed_body.count).to eq(2)
+      end
+    end
+  end
 end