merge origin

mikesndrs · mikesndrs · commit 91858525b3f4 · 2025-03-17T13:23:54.000+01:00
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -458,7 +458,7 @@ GEM
     pundit (2.3.1)
       activesupport (>= 3.0.0)
     racc (1.8.1)
-    rack (2.2.11)
+    rack (2.2.13)
     rack-cors (2.0.2)
       rack (>= 2.0.0)
     rack-oauth2 (2.2.0)
@@ -764,7 +764,7 @@ GEM
     unf_ext (0.0.8.2)
     unicode-display_width (2.4.2)
     unicode-types (1.10.0)
-    uri (0.13.0)
+    uri (0.13.2)
     useragent (0.16.11)
     validate_email (0.1.6)
       activemodel (>= 3.0)
diff --git a/app/controllers/sources_controller.rb b/app/controllers/sources_controller.rb
@@ -150,7 +150,7 @@ def set_content_provider
 
   # Never trust parameters from the scary internet, only allow the white list through.
   def source_params
-    permitted = [:url, :method, :token, :enabled]
+    permitted = [:url, :method, :token, :default_language, :enabled]
     permitted << :approval_status if policy(Source).approve?
     permitted << :content_provider_id if policy(Source).index?
 
diff --git a/app/models/source.rb b/app/models/source.rb
@@ -22,6 +22,8 @@ class Source < ApplicationRecord
   validates :approval_status, inclusion: { in: APPROVAL_STATUS.values }
   validates :method, inclusion: { in: -> (_) { TeSS::Config.user_ingestion_methods } },
             unless: -> { User.current_user&.is_admin? || User.current_user&.has_role?(:scraper_user) }
+  validates :default_language, controlled_vocabulary: { dictionary: 'LanguageDictionary',
+                                                        allow_blank: true }
   validate :check_method
 
   before_create :set_approval_status
diff --git a/app/views/sources/_form.html.erb b/app/views/sources/_form.html.erb
@@ -22,6 +22,11 @@
 
   <%= f.input :token, hint: t('sources.hints.token'), label: 'Authentication Token' %>
 
+  <%= f.input :default_language,
+              collection: LanguageDictionary.instance.options_for_select,
+              prompt: t('sources.prompts.default_language'), include_blank: true,
+              hint: t('sources.hints.default_language') %>
+
   <%= f.input :enabled, hint: t('sources.hints.enabled') %>
 
   <% if policy(@source).approve? %>
diff --git a/app/views/sources/show.html.erb b/app/views/sources/show.html.erb
@@ -52,6 +52,13 @@
                 </p>
               <% end %>
 
+              <!-- Field: default_language -->
+              <% if current_user && (current_user.is_curator? || current_user.is_admin?) %>
+                <p><strong><%= Source.human_attribute_name(:default_language) %>:</strong>
+                  <%= render_language_name(@source.default_language) if @source.default_language %>
+                </p>
+              <% end %>
+
               <!-- Field: enabled -->
               <p>
                 <strong>Source is:</strong>
diff --git a/config/locales/en.yml b/config/locales/en.yml
@@ -64,6 +64,8 @@ en:
           onsite: Face-to-face
           hybrid: Hybrid
         language: 'Language of instruction'
+      source:
+        default_language: 'Default language'
   about:
     headings:
       events: Events
@@ -741,10 +743,13 @@ en:
       method: 'The method used to ingest the contents of the source URL.'
       token: Some ingestion methods require an authentication token, leave blank if you aren't sure.
       enabled: Sources that are not enabled will not be ingested.
+      default_language: 'Default language of ingested events/materials'
     approval_status:
       not_approved: Not approved
       requested: Approval requested
       approved: Approved
+    prompts:
+      default_language: 'Select a default language...'
   scraper:
     messages:
       status: 'Scraper.run: %{status}'
diff --git a/db/migrate/20250303210238_add_default_language_to_source.rb b/db/migrate/20250303210238_add_default_language_to_source.rb
@@ -0,0 +1,5 @@
+class AddDefaultLanguageToSource < ActiveRecord::Migration[7.0]
+  def change
+    add_column :sources, :default_language, :string
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2024_12_02_092029) do
+ActiveRecord::Schema[7.0].define(version: 2025_03_03_210238) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -227,9 +227,9 @@
     t.string "cost_basis"
     t.string "cost_currency"
     t.string "fields", default: [], array: true
-    t.string "open_science", default: [], array: true
     t.boolean "visible", default: true
     t.string "language"
+    t.string "open_science", default: [], array: true
     t.index ["presence"], name: "index_events_on_presence"
     t.index ["slug"], name: "index_events_on_slug", unique: true
     t.index ["user_id"], name: "index_events_on_user_id"
@@ -469,6 +469,7 @@
     t.string "token"
     t.integer "approval_status"
     t.datetime "updated_at"
+    t.string "default_language"
     t.index ["content_provider_id"], name: "index_sources_on_content_provider_id"
     t.index ["user_id"], name: "index_sources_on_user_id"
   end
@@ -572,9 +573,9 @@
     t.string "resource_type"
     t.text "data"
     t.json "params"
-    t.text "referrer"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.text "referrer"
     t.index ["resource_type", "resource_id"], name: "index_widget_logs_on_resource_type_and_resource_id"
   end
 
diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml
@@ -74,6 +74,7 @@ services:
       dockerfile: Dockerfile
       target: production
     image: ${PREFIX}-app
+    restart: always
     depends_on:
       - app
       - db
diff --git a/lib/ingestors/ingestor.rb b/lib/ingestors/ingestor.rb
@@ -33,10 +33,10 @@ def read(_url)
       raise NotImplementedError
     end
 
-    def write(user, provider)
-      write_resources(Event, @events, user, provider)
+    def write(user, provider, source: nil)
+      write_resources(Event, @events, user, provider, source: source)
       @messages << stats_summary(:events)
-      write_resources(Material, @materials, user, provider)
+      write_resources(Material, @materials, user, provider, source: source)
       @messages << stats_summary(:materials)
     end
 
@@ -127,7 +127,7 @@ def set_resource_defaults(resource)
       resource
     end
 
-    def write_resources(type, resources, user, provider)
+    def write_resources(type, resources, user, provider, source: nil)
       resources.each_with_index do |resource, i|
         key = type.model_name.collection.to_sym
         @stats[key][:processed] += 1
@@ -144,6 +144,10 @@ def write_resources(type, resources, user, provider)
                      type.new(resource.to_h)
                    end
 
+        if resource.has_attribute?(:language) && resource.new_record?
+          resource.language ||= source&.default_language
+        end
+
         resource = set_resource_defaults(resource)
         if resource.valid?
           resource.save!
diff --git a/lib/ingestors/oscm_ingestor.rb b/lib/ingestors/oscm_ingestor.rb
@@ -2,14 +2,16 @@
 require 'open-uri'
 require 'csv'
 require 'nokogiri'
+require 'openssl'
 
 module Ingestors
   class OscmIngestor < Ingestor
     def self.config
       {
         key: 'oscm_event',
         title: 'OSCM Events API',
-        category: :events
+        category: :events,
+        ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE
       }
     end
 
@@ -29,9 +31,10 @@ def read(url)
     def process_oscm(url)
       # Instead of using the sitemap we use the events page.
       # The sitemap shows also past events, but the ical link for those does not work, so we can't parse them with the below code.
-      Nokogiri::HTML5(open_url(url, raise: true)).css('.eventname > a').each do |link|
+      url = 'https://www.openscience-maastricht.nl/events/'
+      Nokogiri::HTML5.parse(open_url(url.to_s, raise: true)).css('.eventname > a').each do |event_link|
         begin
-          event_url = link.attributes['href']
+          event_url = event_link.attributes['href']
           event_page = Nokogiri::HTML5.parse(open_url(event_url, raise: true))
 
           # create new event
@@ -43,8 +46,8 @@ def process_oscm(url)
           event.description = convert_description ical_event.description
           event.url = ical_event.url
           # TeSS timezone handling is a bit special.
-          event.start = ical_event.dtstart
-          event.end = ical_event.dtend
+          event.start = Time.zone.parse(ical_event.dtstart.strftime('%a, %d %b %Y %H:%M:%S'))
+          event.end = Time.zone.parse(ical_event.dtend.strftime('%a, %d %b %Y %H:%M:%S'))
           event.set_default_times
           event.venue = ical_event.try(:venue)
           event.keywords = ical_event.categories # fair-coffee pre-registration-workshop fair-essentials-workshop fair-for-qualitative-data reproducibilitea
diff --git a/lib/scraper.rb b/lib/scraper.rb
@@ -133,7 +133,7 @@ def scrape(source, user = get_user, index: 0)
       end
 
       # write resources
-      ingestor.write(user, source.content_provider)
+      ingestor.write(user, source.content_provider, source: source)
       unless ingestor.messages.blank?
         output.concat "\n## Writing\n\n"
         ingestor.messages.each { |m| output.concat "#{m}\n" }
diff --git a/test/unit/ingestors/ingestor_test.rb b/test/unit/ingestors/ingestor_test.rb
@@ -12,4 +12,105 @@ class IngestorTest < ActiveSupport::TestCase
     expected = "# Title\n\n- Item 1\n- Item 2"
     assert_equal expected, ingestor.convert_description(input)
   end
-end
+
+  test 'sets event language from source default language' do
+    user = users(:scraper_user)
+    provider = content_providers(:portal_provider)
+
+    # Source has default language set
+    @source = Source.create!(url: 'https://somewhere.com/stuff', method: 'bioschemas',
+                             enabled: true, approval_status: 'approved',
+                             default_language: 'fr',
+                             content_provider: provider, user: users(:admin))
+
+    ingestor = Ingestors::Ingestor.new
+
+    # Fake an event that was read ... no language set
+    ingestor.instance_variable_set(:@events,
+                                   [OpenStruct.new(url: 'https://some-course.ca',
+                                                   title: 'Some course',
+                                                   start: '2021-01-31 13:00:00',
+                                                   end:'2021-01-31 14:00:00')])
+    assert_difference('provider.events.count', 1) do
+      ingestor.write(user, provider, source: @source)
+    end
+    event = Event.find_by(title: 'Some course')
+    assert_equal(event.language, 'fr')
+  end
+
+  test 'does not override event language from source default language when language set' do
+    user = users(:scraper_user)
+    provider = content_providers(:portal_provider)
+
+    # Source has default language set
+    @source = Source.create!(url: 'https://somewhere.com/stuff', method: 'bioschemas',
+                             enabled: true, approval_status: 'approved',
+                             default_language: 'fr',
+                             content_provider: provider, user: users(:admin))
+
+    ingestor = Ingestors::Ingestor.new
+
+    # Fake an event that was read ... with language set
+    ingestor.instance_variable_set(:@events,
+                                   [OpenStruct.new(url: 'https://some-course.de',
+                                                   title: 'Some german course',
+                                                   start: '2021-01-31 13:00:00',
+                                                   end:'2021-01-31 14:00:00',
+                                                   language: 'de')])
+    assert_difference('provider.events.count', 1) do
+      ingestor.write(user, provider, source: @source)
+    end
+    event = Event.find_by(title: 'Some german course')
+    assert_equal(event.language, 'de')
+  end
+
+  test 'does not override event language when source default language missing' do
+    user = users(:scraper_user)
+    provider = content_providers(:portal_provider)
+
+    # Source has no default language set
+    @source = Source.create!(url: 'https://somewhere.com/stuff', method: 'bioschemas',
+                             enabled: true, approval_status: 'approved',
+                             content_provider: provider, user: users(:admin))
+
+    ingestor = Ingestors::Ingestor.new
+
+    # Fake an event that was read ... with language set
+    ingestor.instance_variable_set(:@events,
+                                   [OpenStruct.new(url: 'https://some-course.org',
+                                                   title: 'Some other course',
+                                                   start: '2021-01-31 13:00:00',
+                                                   end:'2021-01-31 14:00:00',
+                                                   language: 'de')])
+    assert_difference('provider.events.count', 1) do
+      ingestor.write(user, provider, source: @source)
+    end
+    event = Event.find_by(title: 'Some other course')
+    assert_equal(event.language, 'de')
+  end
+
+  test 'does not set event language when languare and source default language missing' do
+    user = users(:scraper_user)
+    provider = content_providers(:portal_provider)
+
+    # Source has no default language set
+    @source = Source.create!(url: 'https://somewhere.com/stuff', method: 'bioschemas',
+                             enabled: true, approval_status: 'approved',
+                             content_provider: provider, user: users(:admin))
+
+    ingestor = Ingestors::Ingestor.new
+
+    # Fake an event that was read ... no language set
+    ingestor.instance_variable_set(:@events,
+                                   [OpenStruct.new(url: 'https://some-course.net',
+                                                   title: 'Yet another course',
+                                                   start: '2021-01-31 13:00:00',
+                                                   end:'2021-01-31 14:00:00')])
+    assert_difference('provider.events.count', 1) do
+      ingestor.write(user, provider, source: @source)
+    end
+    event = Event.find_by(title: 'Yet another course')
+    assert_nil(event.language)
+  end
+
+end
diff --git a/test/unit/ingestors/taxila/oscm_ingestor_test.rb b/test/unit/ingestors/taxila/oscm_ingestor_test.rb
@@ -22,12 +22,12 @@ class OscmIngestorTest < ActiveSupport::TestCase
     ingestor = Ingestors::Taxila::OscmIngestor.new
 
     # check event doesn't
-    new_title = 'FAIR Coffee lecture - Gijs van Dijck'
-    new_url = 'https://www.openscience-maastricht.nl/events/fair-coffee-lecture-24-may-2023/'
+    new_title = 'FAIR Coffee lecture - Mariëlle Prevoo (pre-announcement)'
+    new_url = 'https://www.openscience-maastricht.nl/events/fair-coffee-lecture-marielle-prevoo-pre-announcement/'
     refute Event.where(title: new_title, url: new_url).any?
 
     # run task
-    assert_difference 'Event.count', 3 do
+    assert_difference 'Event.count', 4 do
       freeze_time(2019) do
         VCR.use_cassette("ingestors/oscm") do
           ingestor.read(source.url)
@@ -36,9 +36,9 @@ class OscmIngestorTest < ActiveSupport::TestCase
       end
     end
 
-    assert_equal 3, ingestor.events.count
+    assert_equal 4, ingestor.events.count
     assert ingestor.materials.empty?
-    assert_equal 3, ingestor.stats[:events][:added]
+    assert_equal 4, ingestor.stats[:events][:added]
     assert_equal 0, ingestor.stats[:events][:updated]
     assert_equal 0, ingestor.stats[:events][:rejected]
 
@@ -49,11 +49,11 @@ class OscmIngestorTest < ActiveSupport::TestCase
     assert_equal new_url, event.url
 
     # check other fields
-    assert_equal 'FAIR Coffee lecture - Gijs van Dijck', event.title
+    assert_equal 'FAIR Coffee lecture - Mariëlle Prevoo (pre-announcement)', event.title
     assert_equal 'Amsterdam', event.timezone
     assert_equal 'OSCM', event.source
     assert event.online?
-    assert_equal Time.zone.parse('Wed, 24 May 2023 11:00:00.000000000 UTC +00:00'), event.start
-    assert_equal Time.zone.parse('Wed, 24 May 2023 12:00:00.000000000 UTC +00:00'), event.end
+    assert_equal Time.zone.parse('Wed, 15 May 2025 10:30:00.000000000 UTC +00:00'), event.start
+    assert_equal Time.zone.parse('Wed, 15 May 2025 11:30:00.000000000 UTC +00:00'), event.end
   end
 end
diff --git a/test/vcr_cassettes/ingestors/oscm.yml b/test/vcr_cassettes/ingestors/oscm.yml
