Added possibility to configure app

.gitignore

@@ -1,3 +1,6 @@
+# CUSTOM
+**/config.toml
+
 # Created by https://www.gitignore.io/api/linux,macos,gradle,kotlin,windows,intellij,intellij+all,intellij+iml,visualstudiocode
 # Edit at https://www.gitignore.io/?templates=linux,macos,gradle,kotlin,windows,intellij,intellij+all,intellij+iml,visualstudiocode
 

tooling/app/README.md

@@ -0,0 +1,13 @@
+# modb-extension app
+
+This application creates and updates the data that you can find in the `/data` folder of this repository.
+
+## How to run
+
+1. Setup config by creating a config file `tooling/app/src/main/resources/config.toml` with content:
+```toml
+[modb.extension.config]
+dataDirectory=""
+```
+Set `dataDirectory` to the absolute path of json files on your machine.
+2. Run application via `tooling/app/src/main/kotlin/io/github/manamiproject/modb/extension/Main.kt`

tooling/app/src/main/kotlin/io/github/manamiproject/modb/extension/Main.kt

@@ -1,31 +1,30 @@
 package io.github.manamiproject.modb.extension
 
 import io.github.manamiproject.modb.core.coroutines.CoroutineManager.runCoroutine
-import io.github.manamiproject.modb.core.extensions.Directory
-import io.github.manamiproject.modb.core.extensions.directoryExists
 import io.github.manamiproject.modb.core.extensions.readFile
 import io.github.manamiproject.modb.core.extractor.JsonDataExtractor
 import io.github.manamiproject.modb.core.random
+import io.github.manamiproject.modb.extension.config.AppConfig
+import io.github.manamiproject.modb.extension.config.Config
 import io.github.manamiproject.modb.extension.score.*
 import io.github.manamiproject.modb.extension.synopsis.*
 import io.github.manamiproject.modb.extension.updates.DefaultUpdatableItemsFinder
 import io.github.manamiproject.modb.serde.json.AnimeListJsonStringDeserializer
 import io.github.manamiproject.modb.serde.json.DefaultExternalResourceJsonDeserializer
 import kotlinx.coroutines.delay
 import java.net.URI
-import kotlin.io.path.Path
 import kotlin.io.path.forEachDirectoryEntry
 
 fun main() = runCoroutine {
-    val sourcesFromDb = fetchSourcesFromDb()
-    val dataDirectory = Path(System.getenv("MODB_EXTENSION_DATA_DIR"))
-    val localFileOrigin = LocalFileOrigin(dataDirectory)
-    val existingFiles = fetchAllExistingFiles(dataDirectory)
-    val newDbEntries = DefaultUpdatableItemsFinder(dataDirectory).findNewDbEntries(sourcesFromDb, existingFiles)
+    val appConfig = AppConfig()
+    val sourcesFromDb = fetchSourcesFromDb(appConfig)
+    val localFileOrigin = LocalFileOrigin(appConfig.dataDirectory())
+    val existingFiles = fetchAllExistingFiles(appConfig)
+    val newDbEntries = DefaultUpdatableItemsFinder(appConfig).findNewDbEntries(sourcesFromDb, existingFiles)
 
     val scoreCreator = DefaultScoreCreator()
     val scoreWriter = DefaultScoreWriter(localFileOrigin)
-    val scoreDownloadListCreator = DefaultScoreDownloadListCreator(dataDirectory)
+    val scoreDownloadListCreator = DefaultScoreDownloadListCreator(appConfig)
     val scoreDownloadList = newDbEntries.union(scoreDownloadListCreator.createDownloadList())
     scoreDownloadList.forEachIndexed { index, sourcesBlock ->
         println("Downloading [${index + 1}/${scoreDownloadList.size}]")
@@ -38,7 +37,7 @@ fun main() = runCoroutine {
 
     val synopsisCreator = DefaultSynopsisCreator()
     val synopsisWriter = DefaultSynopsisWriter(localFileOrigin)
-    val synopsisDownloadListCreator = DefaultSynopsisDownloadListCreator(dataDirectory)
+    val synopsisDownloadListCreator = DefaultSynopsisDownloadListCreator(appConfig)
     val synopsisDownloadList = newDbEntries.union(synopsisDownloadListCreator.createDownloadList())
     synopsisDownloadList.forEachIndexed { index, sourcesBlock ->
         println("Downloading [${index + 1}/${synopsisDownloadList.size}]")
@@ -49,20 +48,19 @@ fun main() = runCoroutine {
         }
     }
 
-    check((fetchAllSourcesInFiles(dataDirectory) - sourcesFromDb).isEmpty()) { "All sources in files must exist in db at this point." }
+    check((fetchAllSourcesInFiles(appConfig) - sourcesFromDb).isEmpty()) { "All sources in files must exist in db at this point." }
 
     println("Done")
 }
 
-private suspend fun fetchSourcesFromDb(): Set<HashSet<URI>> {
+private suspend fun fetchSourcesFromDb(config: Config): Set<HashSet<URI>> {
     val deserializer = DefaultExternalResourceJsonDeserializer(deserializer = AnimeListJsonStringDeserializer())
-    val zipFile = URI("https://raw.githubusercontent.com/manami-project/anime-offline-database/master/anime-offline-database.zip")
-    return deserializer.deserialize(zipFile.toURL()).data.map { it.sources }.toSet()
+    return deserializer.deserialize(config.animeDataSet().toURL()).data.map { it.sources }.toSet()
 }
 
-private suspend fun fetchAllSourcesInFiles(directory: Directory): Set<Set<URI>> {
+private suspend fun fetchAllSourcesInFiles(config: Config): Set<Set<URI>> {
     val list = mutableSetOf<Set<URI>>()
-    directory.forEachDirectoryEntry("*.json") { file ->
+    config.dataDirectory().forEachDirectoryEntry("*.json") { file ->
         val extractionResult = JsonDataExtractor.extract(file.readFile(), mapOf(
             "sources" to "$.sources"
         ))
@@ -71,10 +69,9 @@ private suspend fun fetchAllSourcesInFiles(directory: Directory): Set<Set<URI>>
     return list
 }
 
-private fun fetchAllExistingFiles(directory: Directory): Set<String> {
-    require(directory.directoryExists()) { "Data directory either doesn't exist or is not a directory." }
+private fun fetchAllExistingFiles(config: Config): Set<String> {
     val ret = mutableSetOf<String>()
-    directory.forEachDirectoryEntry("*.json") {
+    config.dataDirectory().forEachDirectoryEntry("*.json") {
         ret.add(it.fileName.toString())
     }
     return ret

tooling/app/src/main/kotlin/io/github/manamiproject/modb/extension/config/AppConfig.kt

@@ -0,0 +1,42 @@
+package io.github.manamiproject.modb.extension.config
+
+import io.github.manamiproject.modb.core.config.ConfigRegistry
+import io.github.manamiproject.modb.core.config.DefaultConfigRegistry
+import io.github.manamiproject.modb.core.config.StringPropertyDelegate
+import io.github.manamiproject.modb.core.extensions.Directory
+import io.github.manamiproject.modb.core.extensions.directoryExists
+import java.net.URI
+import kotlin.io.path.Path
+
+/**
+ * Implementation of [Config] which contains all necessary properties.
+ * @since 1.0.0
+ * @param configRegistry Implementation of [ConfigRegistry] used for populating properties. Uses [DefaultConfigRegistry] by default.
+ */
+class AppConfig(
+    configRegistry: ConfigRegistry = DefaultConfigRegistry,
+) : Config {
+
+    private val dataDirectory: String by StringPropertyDelegate(
+        namespace = NAMESPACE,
+        configRegistry = configRegistry,
+    )
+
+    private val animeDataset: String by StringPropertyDelegate(
+        namespace = NAMESPACE,
+        configRegistry = configRegistry,
+        default = "https://raw.githubusercontent.com/manami-project/anime-offline-database/master/anime-offline-database.zip"
+    )
+
+    override fun dataDirectory(): Directory {
+        val path = Path(dataDirectory)
+        check(path.directoryExists()) { "Given value for dataDirectory [$dataDirectory] doesn't exist or is not a directory." }
+        return path
+    }
+
+    override fun animeDataSet(): URI = URI(animeDataset)
+
+    companion object {
+        private const val NAMESPACE = "modb.extension.config"
+    }
+}

tooling/app/src/main/kotlin/io/github/manamiproject/modb/extension/config/Config.kt

@@ -0,0 +1,35 @@
+package io.github.manamiproject.modb.extension.config
+
+import io.github.manamiproject.modb.core.extensions.Directory
+import java.net.URI
+import java.time.Clock
+
+/**
+ * Contains all configurable and directly derived properties.
+ * @since 1.0.0
+ */
+interface Config {
+
+    /**
+     * Returns the directory in which the `*.json` files are being saved.
+     * @since 1.0.0
+     * @return Path to the JSON files created by the app.
+     */
+    fun dataDirectory(): Directory
+
+    /**
+     * Source of the anime data set.
+     * Expected is the format defined [here](https://github.com/manami-project/anime-offline-database).
+     * @since 1.0.0
+     * @return [URI] pointing to the anime data set.
+     */
+    fun animeDataSet(): URI
+
+    /**
+     * Clock being used whenever dates and timestamps are created.
+     * Default is system default zone.
+     * @since 1.0.0
+     * @return Instance of [Clock] as basis for any type of date instances.
+     */
+    fun clock(): Clock = Clock.systemDefaultZone()
+}

tooling/app/src/main/kotlin/io/github/manamiproject/modb/extension/score/DefaultScoreDownloadListCreator.kt

@@ -1,40 +1,30 @@
 package io.github.manamiproject.modb.extension.score
 
-import io.github.manamiproject.modb.core.extensions.Directory
-import io.github.manamiproject.modb.core.extensions.EMPTY
-import io.github.manamiproject.modb.core.extensions.directoryExists
 import io.github.manamiproject.modb.core.extensions.readFile
 import io.github.manamiproject.modb.core.extractor.DataExtractor
 import io.github.manamiproject.modb.core.extractor.JsonDataExtractor
 import io.github.manamiproject.modb.core.json.Json
 import io.github.manamiproject.modb.extension.ExtensionData
-import io.github.manamiproject.modb.extension.filename
+import io.github.manamiproject.modb.extension.config.Config
 import java.net.URI
-import java.time.Clock
 import java.time.LocalDate
 import java.time.Period
 import kotlin.io.path.forEachDirectoryEntry
 
 /**
  * @since 1.0.0
- * @property directory
- * @property clock
+ * @property config
  * @property extractor
  */
 class DefaultScoreDownloadListCreator(
-    private val directory: Directory,
-    private val clock: Clock = Clock.systemDefaultZone(),
+    private val config: Config,
     private val extractor: DataExtractor = JsonDataExtractor,
 ): ScoreDownloadListCreator {
 
-    init {
-        require(directory.directoryExists()) { "Data directory either doesn't exist or is not a directory." }
-    }
-
     override suspend fun createDownloadList(redownloadEntriesOlderThan: Period): Set<HashSet<URI>> {
         val ret = mutableSetOf<HashSet<URI>>()
 
-        directory.forEachDirectoryEntry("*.json") { file ->
+        config.dataDirectory().forEachDirectoryEntry("*.json") { file ->
             val fileContent = file.readFile()
             val extensionData = Json.parseJson<ExtensionData>(fileContent)!!
 
@@ -64,6 +54,6 @@ class DefaultScoreDownloadListCreator(
     }
 
     private fun isRedownloadNecessary(redownloadEntriesOlderThan: Period, dateToCheck: LocalDate): Boolean {
-        return LocalDate.now(clock).minus(redownloadEntriesOlderThan).isAfter(dateToCheck)
+        return LocalDate.now(config.clock()).minus(redownloadEntriesOlderThan).isAfter(dateToCheck)
     }
 }

tooling/app/src/main/kotlin/io/github/manamiproject/modb/extension/synopsis/DefaultSynopsisDownloadListCreator.kt

@@ -1,40 +1,31 @@
 package io.github.manamiproject.modb.extension.synopsis
 
-import io.github.manamiproject.modb.core.extensions.Directory
-import io.github.manamiproject.modb.core.extensions.EMPTY
-import io.github.manamiproject.modb.core.extensions.directoryExists
 import io.github.manamiproject.modb.core.extensions.readFile
 import io.github.manamiproject.modb.core.extractor.DataExtractor
 import io.github.manamiproject.modb.core.extractor.JsonDataExtractor
 import io.github.manamiproject.modb.core.json.Json
 import io.github.manamiproject.modb.extension.ExtensionData
-import io.github.manamiproject.modb.extension.filename
+import io.github.manamiproject.modb.extension.config.Config
 import java.net.URI
-import java.time.Clock
 import java.time.LocalDate
 import java.time.Period
 import kotlin.io.path.forEachDirectoryEntry
 
 /**
  * @since 1.0.0
- * @property directory
+ * @property config
  * @property clock
  * @property extractor
  */
 class DefaultSynopsisDownloadListCreator(
-    private val directory: Directory,
-    private val clock: Clock = Clock.systemDefaultZone(),
+    private val config: Config,
     private val extractor: DataExtractor = JsonDataExtractor,
 ): SynopsisDownloadListCreator {
 
-    init {
-        require(directory.directoryExists()) { "Data directory either doesn't exist or is not a directory." }
-    }
-
     override suspend fun createDownloadList(redownloadEntriesOlderThan: Period): Set<HashSet<URI>> {
         val ret = mutableSetOf<HashSet<URI>>()
 
-        directory.forEachDirectoryEntry("*.json") { file ->
+        config.dataDirectory().forEachDirectoryEntry("*.json") { file ->
             val fileContent = file.readFile()
             val extensionData = Json.parseJson<ExtensionData>(fileContent)!!
             when (val synopsis = extensionData.synopsis()) {
@@ -63,6 +54,6 @@ class DefaultSynopsisDownloadListCreator(
     }
 
     private fun isRedownloadNecessary(redownloadEntriesOlderThan: Period, dateToCheck: LocalDate): Boolean {
-        return LocalDate.now(clock).minus(redownloadEntriesOlderThan).isAfter(dateToCheck)
+        return LocalDate.now(config.clock()).minus(redownloadEntriesOlderThan).isAfter(dateToCheck)
     }
 }

tooling/app/src/main/kotlin/io/github/manamiproject/modb/extension/updates/DefaultUpdatableItemsFinder.kt

@@ -1,20 +1,17 @@
 package io.github.manamiproject.modb.extension.updates
 
-import io.github.manamiproject.modb.core.extensions.Directory
-import io.github.manamiproject.modb.core.extensions.directoryExists
+import io.github.manamiproject.modb.extension.config.Config
 import io.github.manamiproject.modb.extension.filename
 import java.net.URI
 import kotlin.io.path.deleteIfExists
 
 /**
  * @since 1.0.0
- * @property directory
+ * @property config
  */
-class DefaultUpdatableItemsFinder(private val directory: Directory): UpdatableItemsFinder {
-
-    init {
-        require(directory.directoryExists()) { "Data directory either doesn't exist or is not a directory." }
-    }
+class DefaultUpdatableItemsFinder(
+    private val config: Config
+): UpdatableItemsFinder {
 
     override suspend fun findNewDbEntries(
         sourcesFromDb: Set<HashSet<URI>>,
@@ -29,7 +26,7 @@ class DefaultUpdatableItemsFinder(private val directory: Directory): UpdatableIt
         existingFileNames.toMutableSet().apply {
             removeAll(expectedFilenameToDbSources.keys)
         }.forEach {
-            directory.resolve(it).deleteIfExists()
+            config.dataDirectory().resolve(it).deleteIfExists()
         }
 
         return newOrUpdatedEntriesInDb.values.toSet()

tooling/app/src/test/kotlin/io/github/manamiproject/modb/extension/TestingAssets.kt

@@ -1,10 +1,34 @@
 package io.github.manamiproject.modb.extension
 
 import io.github.manamiproject.modb.core.config.AnimeId
+import io.github.manamiproject.modb.core.config.ConfigRegistry
 import io.github.manamiproject.modb.core.downloader.Downloader
+import io.github.manamiproject.modb.core.extensions.Directory
+import io.github.manamiproject.modb.extension.config.Config
 import io.github.manamiproject.modb.test.shouldNotBeInvoked
+import java.net.URI
+import java.time.Clock
+import java.time.LocalDate
+import java.time.LocalDateTime
 
 internal object TestDownloader: Downloader {
     override suspend fun download(id: AnimeId, onDeadEntry: suspend (AnimeId) -> Unit): String = shouldNotBeInvoked()
 }
 
+internal object TestConfigRegistry: ConfigRegistry {
+    override fun boolean(key: String): Boolean = shouldNotBeInvoked()
+    override fun double(key: String): Double = shouldNotBeInvoked()
+    override fun <T : Any> list(key: String): List<T> = shouldNotBeInvoked()
+    override fun localDate(key: String): LocalDate = shouldNotBeInvoked()
+    override fun localDateTime(key: String): LocalDateTime = shouldNotBeInvoked()
+    override fun long(key: String): Long = shouldNotBeInvoked()
+    override fun <T : Any> map(key: String): Map<String, T> = shouldNotBeInvoked()
+    override fun offsetDateTime(key: String) = shouldNotBeInvoked()
+    override fun string(key: String): String = shouldNotBeInvoked()
+}
+
+internal object TestConfig: Config {
+    override fun dataDirectory(): Directory = shouldNotBeInvoked()
+    override fun animeDataSet(): URI = shouldNotBeInvoked()
+    override fun clock(): Clock = shouldNotBeInvoked()
+}