Merge pull request #1081 from cwant/cwant/source_default_language

Source default language

Author: fbacall

app/controllers/sources_controller.rb

@@ -150,7 +150,7 @@ def set_content_provider
 
   # Never trust parameters from the scary internet, only allow the white list through.
   def source_params
-    permitted = [:url, :method, :token, :enabled]
+    permitted = [:url, :method, :token, :default_language, :enabled]
     permitted << :approval_status if policy(Source).approve?
     permitted << :content_provider_id if policy(Source).index?
 

app/models/source.rb

@@ -22,6 +22,8 @@ class Source < ApplicationRecord
   validates :approval_status, inclusion: { in: APPROVAL_STATUS.values }
   validates :method, inclusion: { in: -> (_) { TeSS::Config.user_ingestion_methods } },
             unless: -> { User.current_user&.is_admin? || User.current_user&.has_role?(:scraper_user) }
+  validates :default_language, controlled_vocabulary: { dictionary: 'LanguageDictionary',
+                                                        allow_blank: true }
   validate :check_method
 
   before_create :set_approval_status

app/views/sources/_form.html.erb

@@ -22,6 +22,11 @@
 
   <%= f.input :token, hint: t('sources.hints.token'), label: 'Authentication Token' %>
 
+  <%= f.input :default_language,
+              collection: LanguageDictionary.instance.options_for_select,
+              prompt: t('sources.prompts.default_language'), include_blank: true,
+              hint: t('sources.hints.default_language') %>
+
   <%= f.input :enabled, hint: t('sources.hints.enabled') %>
 
   <% if policy(@source).approve? %>

app/views/sources/show.html.erb

@@ -52,6 +52,13 @@
                 </p>
               <% end %>
 
+              <!-- Field: default_language -->
+              <% if current_user && (current_user.is_curator? || current_user.is_admin?) %>
+                <p><strong><%= Source.human_attribute_name(:default_language) %>:</strong>
+                  <%= render_language_name(@source.default_language) if @source.default_language %>
+                </p>
+              <% end %>
+
               <!-- Field: enabled -->
               <p>
                 <strong>Source is:</strong>

config/locales/en.yml

@@ -64,6 +64,8 @@ en:
           onsite: Face-to-face
           hybrid: Hybrid
         language: 'Language of instruction'
+      source:
+        default_language: 'Default language'
   about:
     headings:
       events: Events
@@ -741,10 +743,13 @@ en:
       method: 'The method used to ingest the contents of the source URL.'
       token: Some ingestion methods require an authentication token, leave blank if you aren't sure.
       enabled: Sources that are not enabled will not be ingested.
+      default_language: 'Default language of ingested events/materials'
     approval_status:
       not_approved: Not approved
       requested: Approval requested
       approved: Approved
+    prompts:
+      default_language: 'Select a default language...'
   scraper:
     messages:
       status: 'Scraper.run: %{status}'

db/migrate/20250303210238_add_default_language_to_source.rb

@@ -0,0 +1,5 @@
+class AddDefaultLanguageToSource < ActiveRecord::Migration[7.0]
+  def change
+    add_column :sources, :default_language, :string
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2024_12_02_092029) do
+ActiveRecord::Schema[7.0].define(version: 2025_03_03_210238) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -227,9 +227,9 @@
     t.string "cost_basis"
     t.string "cost_currency"
     t.string "fields", default: [], array: true
-    t.string "open_science", default: [], array: true
     t.boolean "visible", default: true
     t.string "language"
+    t.string "open_science", default: [], array: true
     t.index ["presence"], name: "index_events_on_presence"
     t.index ["slug"], name: "index_events_on_slug", unique: true
     t.index ["user_id"], name: "index_events_on_user_id"
@@ -469,6 +469,7 @@
     t.string "token"
     t.integer "approval_status"
     t.datetime "updated_at"
+    t.string "default_language"
     t.index ["content_provider_id"], name: "index_sources_on_content_provider_id"
     t.index ["user_id"], name: "index_sources_on_user_id"
   end
@@ -572,9 +573,9 @@
     t.string "resource_type"
     t.text "data"
     t.json "params"
-    t.text "referrer"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.text "referrer"
     t.index ["resource_type", "resource_id"], name: "index_widget_logs_on_resource_type_and_resource_id"
   end
 

lib/ingestors/ingestor.rb

@@ -33,10 +33,10 @@ def read(_url)
       raise NotImplementedError
     end
 
-    def write(user, provider)
-      write_resources(Event, @events, user, provider)
+    def write(user, provider, source: nil)
+      write_resources(Event, @events, user, provider, source: source)
       @messages << stats_summary(:events)
-      write_resources(Material, @materials, user, provider)
+      write_resources(Material, @materials, user, provider, source: source)
       @messages << stats_summary(:materials)
     end
 
@@ -127,7 +127,7 @@ def set_resource_defaults(resource)
       resource
     end
 
-    def write_resources(type, resources, user, provider)
+    def write_resources(type, resources, user, provider, source: nil)
       resources.each_with_index do |resource, i|
         key = type.model_name.collection.to_sym
         @stats[key][:processed] += 1
@@ -144,6 +144,10 @@ def write_resources(type, resources, user, provider)
                      type.new(resource.to_h)
                    end
 
+        if resource.has_attribute?(:language) && resource.new_record?
+          resource.language ||= source&.default_language
+        end
+
         resource = set_resource_defaults(resource)
         if resource.valid?
           resource.save!

lib/scraper.rb

@@ -133,7 +133,7 @@ def scrape(source, user = get_user, index: 0)
       end
 
       # write resources
-      ingestor.write(user, source.content_provider)
+      ingestor.write(user, source.content_provider, source: source)
       unless ingestor.messages.blank?
         output.concat "\n## Writing\n\n"
         ingestor.messages.each { |m| output.concat "#{m}\n" }

test/unit/ingestors/ingestor_test.rb

@@ -12,4 +12,105 @@ class IngestorTest < ActiveSupport::TestCase
     expected = "# Title\n\n- Item 1\n- Item 2"
     assert_equal expected, ingestor.convert_description(input)
   end
-end
+
+  test 'sets event language from source default language' do
+    user = users(:scraper_user)
+    provider = content_providers(:portal_provider)
+
+    # Source has default language set
+    @source = Source.create!(url: 'https://somewhere.com/stuff', method: 'bioschemas',
+                             enabled: true, approval_status: 'approved',
+                             default_language: 'fr',
+                             content_provider: provider, user: users(:admin))
+
+    ingestor = Ingestors::Ingestor.new
+
+    # Fake an event that was read ... no language set
+    ingestor.instance_variable_set(:@events,
+                                   [OpenStruct.new(url: 'https://some-course.ca',
+                                                   title: 'Some course',
+                                                   start: '2021-01-31 13:00:00',
+                                                   end:'2021-01-31 14:00:00')])
+    assert_difference('provider.events.count', 1) do
+      ingestor.write(user, provider, source: @source)
+    end
+    event = Event.find_by(title: 'Some course')
+    assert_equal(event.language, 'fr')
+  end
+
+  test 'does not override event language from source default language when language set' do
+    user = users(:scraper_user)
+    provider = content_providers(:portal_provider)
+
+    # Source has default language set
+    @source = Source.create!(url: 'https://somewhere.com/stuff', method: 'bioschemas',
+                             enabled: true, approval_status: 'approved',
+                             default_language: 'fr',
+                             content_provider: provider, user: users(:admin))
+
+    ingestor = Ingestors::Ingestor.new
+
+    # Fake an event that was read ... with language set
+    ingestor.instance_variable_set(:@events,
+                                   [OpenStruct.new(url: 'https://some-course.de',
+                                                   title: 'Some german course',
+                                                   start: '2021-01-31 13:00:00',
+                                                   end:'2021-01-31 14:00:00',
+                                                   language: 'de')])
+    assert_difference('provider.events.count', 1) do
+      ingestor.write(user, provider, source: @source)
+    end
+    event = Event.find_by(title: 'Some german course')
+    assert_equal(event.language, 'de')
+  end
+
+  test 'does not override event language when source default language missing' do
+    user = users(:scraper_user)
+    provider = content_providers(:portal_provider)
+
+    # Source has no default language set
+    @source = Source.create!(url: 'https://somewhere.com/stuff', method: 'bioschemas',
+                             enabled: true, approval_status: 'approved',
+                             content_provider: provider, user: users(:admin))
+
+    ingestor = Ingestors::Ingestor.new
+
+    # Fake an event that was read ... with language set
+    ingestor.instance_variable_set(:@events,
+                                   [OpenStruct.new(url: 'https://some-course.org',
+                                                   title: 'Some other course',
+                                                   start: '2021-01-31 13:00:00',
+                                                   end:'2021-01-31 14:00:00',
+                                                   language: 'de')])
+    assert_difference('provider.events.count', 1) do
+      ingestor.write(user, provider, source: @source)
+    end
+    event = Event.find_by(title: 'Some other course')
+    assert_equal(event.language, 'de')
+  end
+
+  test 'does not set event language when languare and source default language missing' do
+    user = users(:scraper_user)
+    provider = content_providers(:portal_provider)
+
+    # Source has no default language set
+    @source = Source.create!(url: 'https://somewhere.com/stuff', method: 'bioschemas',
+                             enabled: true, approval_status: 'approved',
+                             content_provider: provider, user: users(:admin))
+
+    ingestor = Ingestors::Ingestor.new
+
+    # Fake an event that was read ... no language set
+    ingestor.instance_variable_set(:@events,
+                                   [OpenStruct.new(url: 'https://some-course.net',
+                                                   title: 'Yet another course',
+                                                   start: '2021-01-31 13:00:00',
+                                                   end:'2021-01-31 14:00:00')])
+    assert_difference('provider.events.count', 1) do
+      ingestor.write(user, provider, source: @source)
+    end
+    event = Event.find_by(title: 'Yet another course')
+    assert_nil(event.language)
+  end
+
+end