Skip to content

Commit 510a38a

Browse files
spacecowboyspacecowboy
committed
Added reading time/word count for languages which use spaces
1 parent e89df3b commit 510a38a

26 files changed

+1989
-29
lines changed

‎app/schemas/com.nononsenseapps.feeder.db.room.AppDatabase/30.json

+743
Large diffs are not rendered by default.

‎app/schemas/com.nononsenseapps.feeder.db.room.AppDatabase/31.json

+749
Large diffs are not rendered by default.

‎app/src/androidTest/java/com/nononsenseapps/feeder/db/room/TestMigrationFrom29To30.kt

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package com.nononsenseapps.feeder.db.room
2+
3+
import androidx.room.testing.MigrationTestHelper
4+
import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory
5+
import androidx.test.core.app.ApplicationProvider
6+
import androidx.test.ext.junit.runners.AndroidJUnit4
7+
import androidx.test.filters.LargeTest
8+
import androidx.test.platform.app.InstrumentationRegistry
9+
import com.nononsenseapps.feeder.FeederApplication
10+
import kotlin.test.assertEquals
11+
import org.junit.Rule
12+
import org.junit.Test
13+
import org.junit.runner.RunWith
14+
import org.kodein.di.DI
15+
import org.kodein.di.DIAware
16+
import org.kodein.di.android.closestDI
17+
18+
@RunWith(AndroidJUnit4::class)
19+
@LargeTest
20+
class TestMigrationFrom29To30 : DIAware {
21+
private val dbName = "testDb"
22+
private val feederApplication: FeederApplication = ApplicationProvider.getApplicationContext()
23+
override val di: DI by closestDI(feederApplication)
24+
25+
@Rule
26+
@JvmField
27+
val testHelper: MigrationTestHelper = MigrationTestHelper(
28+
InstrumentationRegistry.getInstrumentation(),
29+
AppDatabase::class.java,
30+
emptyList(),
31+
FrameworkSQLiteOpenHelperFactory(),
32+
)
33+
34+
@Test
35+
fun migrate() {
36+
@Suppress("SimpleRedundantLet")
37+
testHelper.createDatabase(dbName, FROM_VERSION).let { oldDB ->
38+
oldDB.execSQL(
39+
"""
40+
INSERT INTO feeds(id, title, url, custom_title, tag, notify, last_sync, response_hash, fulltext_by_default, open_articles_with, alternate_id, currently_syncing, when_modified, site_fetched)
41+
VALUES(1, 'feed', 'http://url', '', '', 0, 0, 666, 0, '', 0, 0, 0, 0)
42+
""".trimIndent(),
43+
)
44+
oldDB.execSQL(
45+
"""
46+
INSERT INTO feed_items(id, guid, title, plain_title, plain_snippet, notified, feed_id, first_synced_time, primary_sort_time, pinned, bookmarked, fulltext_downloaded, read_time, unread)
47+
VALUES(8, 'http://item1', 'title', 'ptitle', 'psnippet', 0, 1, 0, 0, 1, 0, 0, 0, 1)
48+
""".trimIndent(),
49+
)
50+
}
51+
val db = testHelper.runMigrationsAndValidate(
52+
dbName,
53+
TO_VERSION,
54+
true,
55+
MigrationFrom29To30(di),
56+
)
57+
58+
db.query(
59+
"""
60+
SELECT word_count FROM feed_items
61+
""".trimIndent(),
62+
).use {
63+
assert(it.count == 1)
64+
assert(it.moveToFirst())
65+
assertEquals(0, it.getInt(0))
66+
}
67+
}
68+
69+
companion object {
70+
private const val FROM_VERSION = 29
71+
private const val TO_VERSION = 30
72+
}
73+
}

‎app/src/androidTest/java/com/nononsenseapps/feeder/db/room/TestMigrationFrom30To31.kt

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package com.nononsenseapps.feeder.db.room
2+
3+
import androidx.room.testing.MigrationTestHelper
4+
import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory
5+
import androidx.test.core.app.ApplicationProvider
6+
import androidx.test.ext.junit.runners.AndroidJUnit4
7+
import androidx.test.filters.LargeTest
8+
import androidx.test.platform.app.InstrumentationRegistry
9+
import com.nononsenseapps.feeder.FeederApplication
10+
import kotlin.test.assertEquals
11+
import org.junit.Rule
12+
import org.junit.Test
13+
import org.junit.runner.RunWith
14+
import org.kodein.di.DI
15+
import org.kodein.di.DIAware
16+
import org.kodein.di.android.closestDI
17+
18+
@RunWith(AndroidJUnit4::class)
19+
@LargeTest
20+
class TestMigrationFrom30To31 : DIAware {
21+
private val dbName = "testDb"
22+
private val feederApplication: FeederApplication = ApplicationProvider.getApplicationContext()
23+
override val di: DI by closestDI(feederApplication)
24+
25+
@Rule
26+
@JvmField
27+
val testHelper: MigrationTestHelper = MigrationTestHelper(
28+
InstrumentationRegistry.getInstrumentation(),
29+
AppDatabase::class.java,
30+
emptyList(),
31+
FrameworkSQLiteOpenHelperFactory(),
32+
)
33+
34+
@Test
35+
fun migrate() {
36+
@Suppress("SimpleRedundantLet")
37+
testHelper.createDatabase(dbName, FROM_VERSION).let { oldDB ->
38+
oldDB.execSQL(
39+
"""
40+
INSERT INTO feeds(id, title, url, custom_title, tag, notify, last_sync, response_hash, fulltext_by_default, open_articles_with, alternate_id, currently_syncing, when_modified, site_fetched)
41+
VALUES(1, 'feed', 'http://url', '', '', 0, 0, 666, 0, '', 0, 0, 0, 0)
42+
""".trimIndent(),
43+
)
44+
oldDB.execSQL(
45+
"""
46+
INSERT INTO feed_items(id, guid, title, plain_title, plain_snippet, notified, feed_id, first_synced_time, primary_sort_time, pinned, bookmarked, fulltext_downloaded, read_time, unread, word_count)
47+
VALUES(8, 'http://item1', 'title', 'ptitle', 'psnippet', 0, 1, 0, 0, 1, 0, 0, 0, 1, 5)
48+
""".trimIndent(),
49+
)
50+
}
51+
val db = testHelper.runMigrationsAndValidate(
52+
dbName,
53+
TO_VERSION,
54+
true,
55+
MigrationFrom30To31(di),
56+
)
57+
58+
db.query(
59+
"""
60+
SELECT word_count_full FROM feed_items
61+
""".trimIndent(),
62+
).use {
63+
assert(it.count == 1)
64+
assert(it.moveToFirst())
65+
assertEquals(0, it.getInt(0))
66+
}
67+
}
68+
69+
companion object {
70+
private const val FROM_VERSION = 30
71+
private const val TO_VERSION = 31
72+
}
73+
}

‎app/src/main/java/com/nononsenseapps/feeder/archmodel/FeedItemStore.kt

+5
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,10 @@ class FeedItemStore(override val di: DI) : DIAware {
290290
dao.deleteFeedItems(ids)
291291
}
292292

293+
suspend fun updateWordCountFull(id: Long, wordCount: Int) {
294+
dao.updateWordCountFull(id, wordCount)
295+
}
296+
293297
companion object {
294298
private const val PAGE_SIZE = 100
295299
}
@@ -315,6 +319,7 @@ private fun PreviewItem.toFeedListItem() =
315319
feedImageUrl = feedImageUrl,
316320
rawPubDate = pubDate,
317321
primarySortTime = primarySortTime,
322+
wordCount = bestWordCount,
318323
)
319324

320325
private fun LocalDateTime.formatDynamically(): String {

‎app/src/main/java/com/nononsenseapps/feeder/archmodel/Repository.kt

+11
Original file line numberDiff line numberDiff line change
@@ -652,6 +652,15 @@ class Repository(override val di: DI) : DIAware {
652652
settingsStore.setOpenAdjacent(value)
653653
}
654654

655+
val showReadingTime: StateFlow<Boolean> = settingsStore.showReadingTime
656+
fun setShowReadingTime(value: Boolean) {
657+
settingsStore.setShowReadingTime(value)
658+
}
659+
660+
suspend fun updateWordCountFull(id: Long, wordCount: Int) {
661+
feedItemStore.updateWordCountFull(id, wordCount)
662+
}
663+
655664
companion object {
656665
private const val LOG_TAG = "FEEDER_REPO"
657666
}
@@ -713,6 +722,8 @@ data class Article(
713722
val feedId: Long = item?.feedId ?: ID_UNSET
714723
val feedUrl: String? = item?.feedUrl?.toString()
715724
val bookmarked: Boolean = item?.bookmarked ?: false
725+
val wordCount: Int = item?.wordCount ?: 0
726+
val wordCountFull: Int = item?.wordCountFull ?: 0
716727
}
717728

718729
enum class TextToDisplay {

‎app/src/main/java/com/nononsenseapps/feeder/archmodel/SettingsStore.kt

+9
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,13 @@ class SettingsStore(override val di: DI) : DIAware {
314314
sp.edit().putBoolean(PREF_OPEN_ADJACENT, value).apply()
315315
}
316316

317+
private val _showReadingTime = MutableStateFlow(sp.getBoolean(PREF_LIST_SHOW_READING_TIME, false))
318+
val showReadingTime = _showReadingTime.asStateFlow()
319+
fun setShowReadingTime(value: Boolean) {
320+
_showReadingTime.value = value
321+
sp.edit().putBoolean(PREF_LIST_SHOW_READING_TIME, value).apply()
322+
}
323+
317324
private val _feedItemStyle = MutableStateFlow(
318325
feedItemStyleFromString(sp.getStringNonNull(PREF_FEED_ITEM_STYLE, FeedItemStyle.CARD.name)),
319326
)
@@ -510,6 +517,8 @@ const val PREFS_FILTER_READ = "prefs_filter_read"
510517

511518
const val PREF_LIST_SHOW_ONLY_TITLES = "prefs_list_show_only_titles"
512519

520+
const val PREF_LIST_SHOW_READING_TIME = "pref_show_reading_time"
521+
513522
/**
514523
* Read Aloud Settings
515524
*/

‎app/src/main/java/com/nononsenseapps/feeder/db/Constants.kt

+2
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ const val COL_GLOB_PATTERN = "glob_pattern"
4747
const val COL_FULLTEXT_DOWNLOADED = "fulltext_downloaded"
4848
const val COL_READ_TIME = "read_time"
4949
const val COL_SITE_FETCHED = "site_fetched"
50+
const val COL_WORD_COUNT = "word_count"
51+
const val COL_WORD_COUNT_FULL = "word_count_full"
5052

5153
// year 5000
5254
val FAR_FUTURE = Instant.ofEpochSecond(95635369646)

‎app/src/main/java/com/nononsenseapps/feeder/db/room/AppDatabase.kt

+23-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ private const val LOG_TAG = "FEEDER_APPDB"
5050
RemoteFeed::class,
5151
SyncDevice::class,
5252
],
53-
version = 29,
53+
version = 31,
5454
)
5555
@TypeConverters(Converters::class)
5656
abstract class AppDatabase : RoomDatabase() {
@@ -116,12 +116,34 @@ fun getAllMigrations(di: DI) = arrayOf(
116116
MigrationFrom26To27(di),
117117
MigrationFrom27To28(di),
118118
MigrationFrom28To29(di),
119+
MigrationFrom29To30(di),
120+
MigrationFrom30To31(di),
119121
)
120122

121123
/*
122124
* 6 represents legacy database
123125
* 7 represents new Room database
124126
*/
127+
class MigrationFrom30To31(override val di: DI) : Migration(30, 31), DIAware {
128+
override fun migrate(database: SupportSQLiteDatabase) {
129+
database.execSQL(
130+
"""
131+
alter table feed_items add column word_count_full integer not null default 0
132+
""".trimIndent(),
133+
)
134+
}
135+
}
136+
137+
class MigrationFrom29To30(override val di: DI) : Migration(29, 30), DIAware {
138+
override fun migrate(database: SupportSQLiteDatabase) {
139+
database.execSQL(
140+
"""
141+
alter table feed_items add column word_count integer not null default 0
142+
""".trimIndent(),
143+
)
144+
}
145+
}
146+
125147
class MigrationFrom28To29(override val di: DI) : Migration(28, 29), DIAware {
126148
override fun migrate(database: SupportSQLiteDatabase) {
127149
database.execSQL(

‎app/src/main/java/com/nononsenseapps/feeder/db/room/FeedItem.kt

+54-8
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ import com.nononsenseapps.feeder.db.COL_PRIMARYSORTTIME
2424
import com.nononsenseapps.feeder.db.COL_PUBDATE
2525
import com.nononsenseapps.feeder.db.COL_READ_TIME
2626
import com.nononsenseapps.feeder.db.COL_TITLE
27+
import com.nononsenseapps.feeder.db.COL_WORD_COUNT
28+
import com.nononsenseapps.feeder.db.COL_WORD_COUNT_FULL
2729
import com.nononsenseapps.feeder.db.FEED_ITEMS_TABLE_NAME
2830
import com.nononsenseapps.feeder.model.host
2931
import com.nononsenseapps.feeder.ui.text.HtmlToPlainTextConverter
@@ -38,6 +40,8 @@ import java.time.ZonedDateTime
3840
const val MAX_TITLE_LENGTH = 200
3941
const val MAX_SNIPPET_LENGTH = 200
4042

43+
private val patternWhitespace = "\\s+".toRegex()
44+
4145
@Entity(
4246
tableName = FEED_ITEMS_TABLE_NAME,
4347
indices = [
@@ -72,21 +76,38 @@ data class FeedItem @Ignore constructor(
7276
@ColumnInfo(name = COL_ENCLOSURELINK) var enclosureLink: String? = null,
7377
@ColumnInfo(name = COL_ENCLOSURE_TYPE) var enclosureType: String? = null,
7478
@ColumnInfo(name = COL_AUTHOR) var author: String? = null,
75-
@ColumnInfo(name = COL_PUBDATE, typeAffinity = ColumnInfo.TEXT) override var pubDate: ZonedDateTime? = null,
79+
@ColumnInfo(
80+
name = COL_PUBDATE,
81+
typeAffinity = ColumnInfo.TEXT,
82+
) override var pubDate: ZonedDateTime? = null,
7683
@ColumnInfo(name = COL_LINK) override var link: String? = null,
77-
@Deprecated("This column has been 'removed' but sqlite doesn't support drop column.", replaceWith = ReplaceWith("readTime"))
84+
@Deprecated(
85+
"This column has been 'removed' but sqlite doesn't support drop column.",
86+
replaceWith = ReplaceWith("readTime"),
87+
)
7888
@ColumnInfo(name = "unread")
7989
var oldUnread: Boolean = true,
8090
@ColumnInfo(name = COL_NOTIFIED) var notified: Boolean = false,
8191
@ColumnInfo(name = COL_FEEDID) var feedId: Long? = null,
82-
@ColumnInfo(name = COL_FIRSTSYNCEDTIME, typeAffinity = ColumnInfo.INTEGER) var firstSyncedTime: Instant = Instant.EPOCH,
83-
@ColumnInfo(name = COL_PRIMARYSORTTIME, typeAffinity = ColumnInfo.INTEGER) override var primarySortTime: Instant = Instant.EPOCH,
92+
@ColumnInfo(
93+
name = COL_FIRSTSYNCEDTIME,
94+
typeAffinity = ColumnInfo.INTEGER,
95+
) var firstSyncedTime: Instant = Instant.EPOCH,
96+
@ColumnInfo(
97+
name = COL_PRIMARYSORTTIME,
98+
typeAffinity = ColumnInfo.INTEGER,
99+
) override var primarySortTime: Instant = Instant.EPOCH,
84100
@Deprecated("This column has been 'removed' but sqlite doesn't support drop column.")
85101
@ColumnInfo(name = "pinned")
86102
var oldPinned: Boolean = false,
87103
@ColumnInfo(name = COL_BOOKMARKED) var bookmarked: Boolean = false,
88104
@ColumnInfo(name = COL_FULLTEXT_DOWNLOADED) var fullTextDownloaded: Boolean = false,
89-
@ColumnInfo(name = COL_READ_TIME, typeAffinity = ColumnInfo.INTEGER) var readTime: Instant? = null,
105+
@ColumnInfo(
106+
name = COL_READ_TIME,
107+
typeAffinity = ColumnInfo.INTEGER,
108+
) var readTime: Instant? = null,
109+
@ColumnInfo(name = COL_WORD_COUNT) var wordCount: Int = 0,
110+
@ColumnInfo(name = COL_WORD_COUNT_FULL) var wordCountFull: Int = 0,
90111
) : FeedItemForFetching, FeedItemCursor {
91112

92113
constructor() : this(id = ID_UNSET)
@@ -101,10 +122,17 @@ data class FeedItem @Ignore constructor(
101122
) {
102123
val converter = HtmlToPlainTextConverter()
103124
// Be careful about nulls.
104-
val text = entry.content_html ?: entry.content_text ?: ""
125+
val plainText = converter.convert(
126+
entry.content_html
127+
?: entry.content_text
128+
?: "",
129+
)
130+
this.wordCount = estimateWordCount(plainText)
131+
105132
val summary: String = (
106-
entry.summary ?: entry.content_text
107-
?: converter.convert(text)
133+
entry.summary
134+
?: entry.content_text
135+
?: plainText
108136
).take(MAX_SNIPPET_LENGTH)
109137

110138
// Make double sure no base64 images are used as thumbnails
@@ -117,6 +145,7 @@ data class FeedItem @Ignore constructor(
117145
feed.feed_url != null && safeImage != null -> {
118146
relativeLinkIntoAbsolute(sloppyLinkToStrictURL(feed.feed_url), safeImage)
119147
}
148+
120149
else -> safeImage
121150
}
122151

@@ -178,3 +207,20 @@ interface FeedItemCursor {
178207
val pubDate: ZonedDateTime?
179208
val id: Long
180209
}
210+
211+
/**
212+
* If language doesn't use spaces, then this function will try to return 0
213+
*/
214+
fun estimateWordCount(plainText: String): Int {
215+
val charCount = plainText.length.toFloat()
216+
val wordCount = plainText.splitToSequence(patternWhitespace).count()
217+
218+
// Calculate average length of chars between spaces
219+
// A typical value for english is 5-7
220+
// A typical value for japanese is 50-80
221+
return if (charCount / wordCount < 15.0) {
222+
wordCount
223+
} else {
224+
0
225+
}
226+
}

‎app/src/main/java/com/nononsenseapps/feeder/db/room/FeedItemDao.kt

+9
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,15 @@ interface FeedItemDao {
4545
)
4646
suspend fun deleteFeedItems(ids: List<Long>): Int
4747

48+
@Query(
49+
"""
50+
update feed_items
51+
set word_count_full = :wordCount
52+
where id = :id
53+
""",
54+
)
55+
suspend fun updateWordCountFull(id: Long, wordCount: Int)
56+
4857
@Query(
4958
"""
5059
SELECT id FROM feed_items

0 commit comments

Comments
 (0)