From 3861fb6b97bf64b2383ee9d65c84c7ba999a3a31 Mon Sep 17 00:00:00 2001 From: Hubert Van De Walle Date: Wed, 19 Aug 2020 05:08:27 +0200 Subject: [PATCH 1/8] Add search module --- pom.xml | 1 + search/pom.xml | 50 +++++++ search/src/main/kotlin/Constants.kt | 7 + search/src/main/kotlin/Extractors.kt | 35 +++++ search/src/main/kotlin/FieldConverters.kt | 26 ++++ search/src/main/kotlin/NoteSearcher.kt | 123 ++++++++++++++++ search/src/test/kotlin/NoteSearcherTest.kt | 162 +++++++++++++++++++++ 7 files changed, 404 insertions(+) create mode 100644 search/pom.xml create mode 100644 search/src/main/kotlin/Constants.kt create mode 100644 search/src/main/kotlin/Extractors.kt create mode 100644 search/src/main/kotlin/FieldConverters.kt create mode 100644 search/src/main/kotlin/NoteSearcher.kt create mode 100644 search/src/test/kotlin/NoteSearcherTest.kt diff --git a/pom.xml b/pom.xml index 137781c..53b4e21 100644 --- a/pom.xml +++ b/pom.xml @@ -10,6 +10,7 @@ app domain shared + search pom diff --git a/search/pom.xml b/search/pom.xml new file mode 100644 index 0000000..4fb947f --- /dev/null +++ b/search/pom.xml @@ -0,0 +1,50 @@ + + + + parent + be.simplenotes + 1.0-SNAPSHOT + + 4.0.0 + + search + + + 8.5.2 + + + + + be.simplenotes + domain + 1.0-SNAPSHOT + + + org.apache.lucene + lucene-core + ${lucene.version} + + + org.apache.lucene + lucene-analyzers-common + ${lucene.version} + + + org.apache.lucene + lucene-queryparser + ${lucene.version} + + + + be.simplenotes + shared + 1.0-SNAPSHOT + test-jar + test + + + + + diff --git a/search/src/main/kotlin/Constants.kt b/search/src/main/kotlin/Constants.kt new file mode 100644 index 0000000..c81f32a --- /dev/null +++ b/search/src/main/kotlin/Constants.kt @@ -0,0 +1,7 @@ +package be.simplenotes.search + +internal const val uuidField = "uuid" +internal const val titleField = "title" +internal const val tagsField = "tags" +internal const val contentField = "content" +internal const val updatedAtField = "updatedAt" diff --git a/search/src/main/kotlin/Extractors.kt b/search/src/main/kotlin/Extractors.kt new file mode 100644 index 0000000..ae1db61 --- /dev/null +++ b/search/src/main/kotlin/Extractors.kt @@ -0,0 +1,35 @@ +package be.simplenotes.search + +import be.simplenotes.domain.model.PersistedNote +import be.simplenotes.domain.model.PersistedNoteMetadata +import org.apache.lucene.document.Document +import org.apache.lucene.document.Field +import org.apache.lucene.document.StringField +import org.apache.lucene.document.TextField +import org.apache.lucene.search.IndexSearcher +import org.apache.lucene.search.TopDocs + +internal fun PersistedNote.toDocument(): Document { + val note = this + return Document().apply { + // non searchable fields + add(StringField(uuidField, UuidFieldConverter.toDoc(note.uuid), Field.Store.YES)) + add(StringField(updatedAtField, LocalDateTimeFieldConverter.toDoc(note.updatedAt), Field.Store.YES)) + + // searchable fields + add(TextField(titleField, note.meta.title, Field.Store.YES)) + add(TextField(tagsField, TagsFieldConverter.toDoc(note.meta.tags), Field.Store.YES)) + add(TextField(contentField, note.html, Field.Store.YES)) + } +} + +internal fun TopDocs.toResults(searcher: IndexSearcher) = scoreDocs.map { + searcher.doc(it.doc).let { doc -> + PersistedNoteMetadata( + title = doc.get(titleField), + uuid = UuidFieldConverter.fromDoc(doc.get(uuidField)), + updatedAt = LocalDateTimeFieldConverter.fromDoc(doc.get(updatedAtField)), + tags = TagsFieldConverter.fromDoc(doc.get(tagsField)) + ) + } +} diff --git a/search/src/main/kotlin/FieldConverters.kt b/search/src/main/kotlin/FieldConverters.kt new file mode 100644 index 0000000..34e3afa --- /dev/null +++ b/search/src/main/kotlin/FieldConverters.kt @@ -0,0 +1,26 @@ +package be.simplenotes.search + +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import java.util.* + +internal interface FieldConverter { + fun toDoc(value: T): String + fun fromDoc(value: String): T +} + +internal object LocalDateTimeFieldConverter : FieldConverter { + private val formatter = DateTimeFormatter.ISO_DATE_TIME + override fun toDoc(value: LocalDateTime): String = formatter.format(value) + override fun fromDoc(value: String): LocalDateTime = LocalDateTime.parse(value, formatter) +} + +internal object UuidFieldConverter : FieldConverter { + override fun toDoc(value: UUID): String = value.toString() + override fun fromDoc(value: String): UUID = UUID.fromString(value) +} + +internal object TagsFieldConverter : FieldConverter> { + override fun toDoc(value: List): String = value.joinToString(" ") + override fun fromDoc(value: String): List = value.split(" ") +} diff --git a/search/src/main/kotlin/NoteSearcher.kt b/search/src/main/kotlin/NoteSearcher.kt new file mode 100644 index 0000000..5c6126a --- /dev/null +++ b/search/src/main/kotlin/NoteSearcher.kt @@ -0,0 +1,123 @@ +package be.simplenotes.search + +import be.simplenotes.domain.model.PersistedNote +import be.simplenotes.domain.model.PersistedNoteMetadata +import org.apache.lucene.analysis.standard.StandardAnalyzer +import org.apache.lucene.index.* +import org.apache.lucene.search.* +import org.apache.lucene.store.Directory +import org.apache.lucene.store.FSDirectory +import org.slf4j.LoggerFactory +import java.io.File +import java.io.IOException +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes +import java.util.* + +data class SearchTerms(val title: String?, val tag: String?, val content: String?) + +class NoteSearcher(basePath: Path = Path.of("/tmp", "lucene")) { + private val baseFile = basePath.toFile() + + private val logger = LoggerFactory.getLogger(javaClass) + + // region utils + private fun getDirectory(userId: Int): Directory { + val index = File(baseFile, userId.toString()).toPath() + return FSDirectory.open(index) + } + + private fun getIndexSearcher(userId: Int): IndexSearcher { + val directory = getDirectory(userId) + val reader: IndexReader = DirectoryReader.open(directory) + return IndexSearcher(reader) + } + // endregion + + fun indexNote(userId: Int, note: PersistedNote) { + logger.debug("Indexing note ${note.uuid} for user $userId") + + val dir = getDirectory(userId) + val config = IndexWriterConfig(StandardAnalyzer()) + val writer = IndexWriter(dir, config) + val doc = note.toDocument() + + with(writer) { + addDocument(doc) + commit() + close() + } + } + + fun deleteIndex(userId: Int, uuid: UUID) { + logger.debug("Deleting indexing $uuid for user $userId") + + val dir = getDirectory(userId) + val config = IndexWriterConfig(StandardAnalyzer()) + val writer = IndexWriter(dir, config) + + with(writer) { + deleteDocuments(TermQuery(Term(uuidField, UuidFieldConverter.toDoc(uuid)))) + commit() + close() + } + } + + fun updateIndex(userId: Int, note: PersistedNote) { + logger.debug("Updating note ${note.uuid} for user $userId") + deleteIndex(userId, note.uuid) + indexNote(userId, note) + } + + fun search(userId: Int, terms: SearchTerms): List { + val searcher = getIndexSearcher(userId) + + val builder = BooleanQuery.Builder() + + terms.title?.let { + val titleQuery = FuzzyQuery(Term(titleField, it)) + builder.add(BooleanClause(titleQuery, BooleanClause.Occur.SHOULD)) + } + + terms.tag?.let { + val tagsQuery = FuzzyQuery(Term(tagsField, it)) + builder.add(BooleanClause(tagsQuery, BooleanClause.Occur.SHOULD)) + } + + terms.content?.let { + val contentQuery = FuzzyQuery(Term(contentField, it)) + builder.add(BooleanClause(contentQuery, BooleanClause.Occur.SHOULD)) + } + + val query = builder.build() + logger.debug("Searching: $query") + + val topDocs = searcher.search(query, 10) + return topDocs.toResults(searcher) + } + + fun dropIndex(userId: Int) { + val index = File(baseFile, userId.toString()).toPath() + try { + Files.walkFileTree( + index, + object : SimpleFileVisitor() { + override fun visitFile(file: Path, attrs: BasicFileAttributes?): FileVisitResult { + Files.delete(file) + return FileVisitResult.CONTINUE + } + + override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult { + Files.delete(dir) + return FileVisitResult.CONTINUE + } + } + ) + } catch (e: IOException) { + // This is fine + } + } +} diff --git a/search/src/test/kotlin/NoteSearcherTest.kt b/search/src/test/kotlin/NoteSearcherTest.kt new file mode 100644 index 0000000..357f1d5 --- /dev/null +++ b/search/src/test/kotlin/NoteSearcherTest.kt @@ -0,0 +1,162 @@ +package be.simplenotes.search + +import be.simplenotes.domain.model.NoteMetadata +import be.simplenotes.domain.model.PersistedNote +import be.simplenotes.domain.model.PersistedNoteMetadata +import org.assertj.core.api.Assertions.assertThat +import org.intellij.lang.annotations.Language +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.parallel.ResourceLock +import java.time.LocalDateTime +import java.util.* + +@ResourceLock("lucene") +internal class NoteSearcherTest { + + // region setup + private val searcher = NoteSearcher() + + private fun index( + title: String, + tags: List = emptyList(), + content: String = "", + uuid: UUID = UUID.randomUUID(), + ): PersistedNote { + val note = PersistedNote(NoteMetadata(title, tags), markdown = "", content, LocalDateTime.now(), uuid) + searcher.indexNote(1, note) + return note + } + + private fun search( + title: String? = null, + tag: String? = null, + content: String? = null, + ): List = searcher.search(1, SearchTerms(title, tag, content)) + + @BeforeEach + @AfterAll + fun dropIndexes() { + searcher.dropIndex(1) + } + // endregion + + @Test + fun `exact title search`() { + index("first") + index("second") + index("flip") + + assertThat(search("first")) + .hasSizeGreaterThanOrEqualTo(1) + .anyMatch { it.title == "first" } + + assertThat(search("nothing")).isEmpty() + } + + @Test + fun `fuzzy title search`() { + index("first") + index("second") + index("flip") + + assertThat(search("firt")) + .hasSizeGreaterThanOrEqualTo(1) + .anyMatch { it.title == "first" } + + assertThat(search("nothing")).isEmpty() + } + + @Test + fun `exact tags search`() { + index("first", tags = listOf("example", "flamingo")) + index("second", tags = listOf("yes")) + index("second") + + assertThat(search(tag = "example")) + .hasSize(1) + .anyMatch { it.title == "first" } + } + + @Test + fun `exact content search`() { + @Language("html") + val content = + """ +
+

Apache Lucene Core

+

Apache LuceneTM is a + high-performance, full-featured text search engine library written entirely in Java. + It is a technology suitable for nearly any application that requires full-text search, + especially cross-platform.

+

Apache Lucene is an open source project available for free download. Please use the + links on the right to access Lucene.

+

LuceneTM Features

+

Lucene offers powerful features through a simple API:

+

Scalable, High-Performance Indexing

+
    +
  • over 150GB/hour on modern hardware
    • +
  • small RAM requirements -- only 1MB heap
    • +
  • incremental indexing as fast as batch indexing
    • +
  • index size roughly 20-30% the size of text indexed
    • +
+ """.trimIndent() + + index("first", content = content) + + assertThat(search(content = "fast")) + .hasSize(1) + .anyMatch { it.title == "first" } + + @Suppress("SpellCheckingInspection") + assertThat(search(content = "preformance")) // <- note the error + .hasSize(1) + .anyMatch { it.title == "first" } + } + + @Test + fun `combined search`() { + @Language("html") + val content = + """ +
+

Apache Lucene Core

+

Apache LuceneTM is a + high-performance, full-featured text search engine library written entirely in Java. + It is a technology suitable for nearly any application that requires full-text search, + especially cross-platform.

+

Apache Lucene is an open source project available for free download. Please use the + links on the right to access Lucene.

+

LuceneTM Features

+

Lucene offers powerful features through a simple API:

+

Scalable, High-Performance Indexing

+
    +
  • over 150GB/hour on modern hardware
    • +
  • small RAM requirements -- only 1MB heap
    • +
  • incremental indexing as fast as batch indexing
    • +
  • index size roughly 20-30% the size of text indexed
    • +
+ """.trimIndent() + + index("first", content = content, tags = listOf("abc")) + + assertThat(search(title = "fir", tag = "abc", content = "20")) + .hasSize(1) + } + + @Test + fun `delete index`() { + val uuid = index("first").uuid + searcher.deleteIndex(1, uuid) + assertThat(search("first")).isEmpty() + } + + @Test + fun `update index`() { + val note = index("first") + searcher.updateIndex(1, note.copy(meta = note.meta.copy(title = "new"))) + assertThat(search("first")).isEmpty() + assertThat(search("new")).hasSize(1) + } +} From ab3766b8b8d44d78d064a7e8540714338f703b6c Mon Sep 17 00:00:00 2001 From: Hubert Van De Walle Date: Wed, 19 Aug 2020 05:20:17 +0200 Subject: [PATCH 2/8] Add searcher to note service --- domain/pom.xml | 5 +++ domain/src/main/kotlin/DomainModule.kt | 2 +- .../src/main/kotlin/usecases/NoteService.kt | 45 ++++++++++++++----- search/src/main/kotlin/SeachModule.kt | 7 +++ 4 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 search/src/main/kotlin/SeachModule.kt diff --git a/domain/pom.xml b/domain/pom.xml index 82f582d..08682fe 100644 --- a/domain/pom.xml +++ b/domain/pom.xml @@ -14,6 +14,11 @@ shared 1.0-SNAPSHOT + + be.simplenotes + search + 1.0-SNAPSHOT + be.simplenotes shared diff --git a/domain/src/main/kotlin/DomainModule.kt b/domain/src/main/kotlin/DomainModule.kt index 2af4176..37f3180 100644 --- a/domain/src/main/kotlin/DomainModule.kt +++ b/domain/src/main/kotlin/DomainModule.kt @@ -26,7 +26,7 @@ val domainModule = module { single { BcryptPasswordHash() } single { SimpleJwt(get()) } single { JwtPayloadExtractor(get()) } - single { NoteService(get(), get()) } + single { NoteService(get(), get(), get()) } single { MarkdownConverterImpl() } single { ExportUseCaseImpl(get()) } } diff --git a/domain/src/main/kotlin/usecases/NoteService.kt b/domain/src/main/kotlin/usecases/NoteService.kt index 196db02..6cf4748 100644 --- a/domain/src/main/kotlin/usecases/NoteService.kt +++ b/domain/src/main/kotlin/usecases/NoteService.kt @@ -1,6 +1,7 @@ package be.simplenotes.domain.usecases import arrow.core.Either +import arrow.core.extensions.fx import be.simplenotes.domain.model.Note import be.simplenotes.domain.model.PersistedNote import be.simplenotes.domain.model.PersistedNoteMetadata @@ -8,33 +9,41 @@ import be.simplenotes.domain.security.HtmlSanitizer import be.simplenotes.domain.usecases.markdown.MarkdownConverter import be.simplenotes.domain.usecases.markdown.MarkdownParsingError import be.simplenotes.domain.usecases.repositories.NoteRepository +import be.simplenotes.search.NoteSearcher import java.util.* class NoteService( private val markdownConverter: MarkdownConverter, private val noteRepository: NoteRepository, + private val searcher: NoteSearcher, ) { - fun create(userId: Int, markdownText: String): Either = - markdownConverter - .renderDocument(markdownText) + fun create(userId: Int, markdownText: String) = Either.fx { + val persistedNote = !markdownConverter.renderDocument(markdownText) .map { it.copy(html = HtmlSanitizer.sanitize(it.html)) } .map { Note(it.metadata, markdown = markdownText, html = it.html) } .map { noteRepository.create(userId, it) } - fun update(userId: Int, uuid: UUID, markdownText: String): Either = - markdownConverter - .renderDocument(markdownText) + searcher.indexNote(userId, persistedNote) + persistedNote + } + + fun update(userId: Int, uuid: UUID, markdownText: String) = Either.fx { + val persistedNote = !markdownConverter.renderDocument(markdownText) .map { it.copy(html = HtmlSanitizer.sanitize(it.html)) } .map { Note(it.metadata, markdown = markdownText, html = it.html) } .map { noteRepository.update(userId, uuid, it) } + persistedNote?.let { searcher.updateIndex(userId, it) } + persistedNote + } + fun paginatedNotes( userId: Int, page: Int, itemsPerPage: Int = 20, tag: String? = null, - deleted: Boolean = false + deleted: Boolean = false, ): PaginatedNotes { val count = noteRepository.count(userId, tag, deleted) val offset = (page - 1) * itemsPerPage @@ -44,9 +53,25 @@ class NoteService( } fun find(userId: Int, uuid: UUID) = noteRepository.find(userId, uuid) - fun trash(userId: Int, uuid: UUID) = noteRepository.delete(userId, uuid, permanent = false) - fun restore(userId: Int, uuid: UUID) = noteRepository.restore(userId, uuid) - fun delete(userId: Int, uuid: UUID) = noteRepository.delete(userId, uuid, permanent = true) + + fun trash(userId: Int, uuid: UUID): Boolean { + val res = noteRepository.delete(userId, uuid, permanent = false) + if (res) searcher.deleteIndex(userId, uuid) + return res + } + + fun restore(userId: Int, uuid: UUID): Boolean { + val res = noteRepository.restore(userId, uuid) + if (res) find(userId, uuid)?.let { note -> searcher.indexNote(userId, note) } + return res + } + + fun delete(userId: Int, uuid: UUID): Boolean { + val res = noteRepository.delete(userId, uuid, permanent = true) + if (res) searcher.deleteIndex(userId, uuid) + return res + } + fun countDeleted(userId: Int) = noteRepository.count(userId, deleted = true) } diff --git a/search/src/main/kotlin/SeachModule.kt b/search/src/main/kotlin/SeachModule.kt new file mode 100644 index 0000000..28f7f90 --- /dev/null +++ b/search/src/main/kotlin/SeachModule.kt @@ -0,0 +1,7 @@ +package be.simplenotes.search + +import org.koin.dsl.module + +val searchModule = module { + single { NoteSearcher() } +} From 12619f65500a546b4f8e0cbf21c30f731ef82e9a Mon Sep 17 00:00:00 2001 From: Hubert Van De Walle Date: Wed, 19 Aug 2020 16:45:55 +0200 Subject: [PATCH 3/8] Index all notes at start --- .gitignore | 3 +++ app/src/main/kotlin/SimpleNotes.kt | 6 ++++++ domain/src/main/kotlin/DomainModule.kt | 2 +- .../src/main/kotlin/usecases/NoteService.kt | 9 ++++++++ .../usecases/repositories/NoteRepository.kt | 1 + .../usecases/repositories/UserRepository.kt | 1 + .../main/kotlin/notes/NoteRepositoryImpl.kt | 21 +++++++++++++++++++ .../main/kotlin/users/UserRepositoryImpl.kt | 1 + search/src/main/kotlin/NoteSearcher.kt | 15 +++++++++++++ search/src/main/kotlin/SeachModule.kt | 3 ++- 10 files changed, 60 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index baa62e3..fa745c3 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,6 @@ app/src/main/resources/static/styles* # h2 db *.db + +# lucene index +.lucene/ diff --git a/app/src/main/kotlin/SimpleNotes.kt b/app/src/main/kotlin/SimpleNotes.kt index 984fefe..8bd5060 100644 --- a/app/src/main/kotlin/SimpleNotes.kt +++ b/app/src/main/kotlin/SimpleNotes.kt @@ -14,8 +14,10 @@ import be.simplenotes.app.views.NoteView import be.simplenotes.app.views.SettingView import be.simplenotes.app.views.UserView import be.simplenotes.domain.domainModule +import be.simplenotes.domain.usecases.NoteService import be.simplenotes.persistance.DbMigrations import be.simplenotes.persistance.persistanceModule +import be.simplenotes.search.searchModule import be.simplenotes.shared.config.DataSourceConfig import be.simplenotes.shared.config.JwtConfig import org.http4k.core.RequestContexts @@ -36,6 +38,7 @@ fun main() { baseModule, noteModule, settingsModule, + searchModule, ) }.koin @@ -50,6 +53,9 @@ fun main() { val migrations = koin.get() migrations.migrate() + val noteService = koin.get() + noteService.indexAll() + koin.get().start() } diff --git a/domain/src/main/kotlin/DomainModule.kt b/domain/src/main/kotlin/DomainModule.kt index 37f3180..6d4de57 100644 --- a/domain/src/main/kotlin/DomainModule.kt +++ b/domain/src/main/kotlin/DomainModule.kt @@ -26,7 +26,7 @@ val domainModule = module { single { BcryptPasswordHash() } single { SimpleJwt(get()) } single { JwtPayloadExtractor(get()) } - single { NoteService(get(), get(), get()) } + single { NoteService(get(), get(), get(), get()) } single { MarkdownConverterImpl() } single { ExportUseCaseImpl(get()) } } diff --git a/domain/src/main/kotlin/usecases/NoteService.kt b/domain/src/main/kotlin/usecases/NoteService.kt index 6cf4748..d28d9ef 100644 --- a/domain/src/main/kotlin/usecases/NoteService.kt +++ b/domain/src/main/kotlin/usecases/NoteService.kt @@ -9,12 +9,14 @@ import be.simplenotes.domain.security.HtmlSanitizer import be.simplenotes.domain.usecases.markdown.MarkdownConverter import be.simplenotes.domain.usecases.markdown.MarkdownParsingError import be.simplenotes.domain.usecases.repositories.NoteRepository +import be.simplenotes.domain.usecases.repositories.UserRepository import be.simplenotes.search.NoteSearcher import java.util.* class NoteService( private val markdownConverter: MarkdownConverter, private val noteRepository: NoteRepository, + private val userRepository: UserRepository, private val searcher: NoteSearcher, ) { @@ -74,6 +76,13 @@ class NoteService( fun countDeleted(userId: Int) = noteRepository.count(userId, deleted = true) + fun indexAll() { + val userIds = userRepository.findAll() + userIds.forEach { id -> + val notes = noteRepository.findAllDetails(id) + searcher.indexNotes(id, notes) + } + } } data class PaginatedNotes(val pages: Int, val notes: List) diff --git a/domain/src/main/kotlin/usecases/repositories/NoteRepository.kt b/domain/src/main/kotlin/usecases/repositories/NoteRepository.kt index 7b5d58f..e961c9b 100644 --- a/domain/src/main/kotlin/usecases/repositories/NoteRepository.kt +++ b/domain/src/main/kotlin/usecases/repositories/NoteRepository.kt @@ -27,4 +27,5 @@ interface NoteRepository { fun find(userId: Int, uuid: UUID): PersistedNote? fun update(userId: Int, uuid: UUID, note: Note): PersistedNote? fun export(userId: Int): List + fun findAllDetails(userId: Int): List } diff --git a/domain/src/main/kotlin/usecases/repositories/UserRepository.kt b/domain/src/main/kotlin/usecases/repositories/UserRepository.kt index 2952e6f..1f52f34 100644 --- a/domain/src/main/kotlin/usecases/repositories/UserRepository.kt +++ b/domain/src/main/kotlin/usecases/repositories/UserRepository.kt @@ -10,4 +10,5 @@ interface UserRepository { fun exists(username: String): Boolean fun exists(id: Int): Boolean fun delete(id: Int): Boolean + fun findAll(): List } diff --git a/persistance/src/main/kotlin/notes/NoteRepositoryImpl.kt b/persistance/src/main/kotlin/notes/NoteRepositoryImpl.kt index 7ab1a30..661cdf6 100644 --- a/persistance/src/main/kotlin/notes/NoteRepositoryImpl.kt +++ b/persistance/src/main/kotlin/notes/NoteRepositoryImpl.kt @@ -197,4 +197,25 @@ internal class NoteRepositoryImpl(private val db: Database) : NoteRepository { ) } } + + override fun findAllDetails(userId: Int): List { + val notes = db.notes + .filterColumns { it.columns - it.deleted } + .filter { (it.userId eq userId) and (it.deleted eq false) } + .toList() + + if (notes.isEmpty()) return emptyList() + + val uuids = notes.map { note -> note.uuid } + + val tagsByUuid = db.tags + .filterColumns { listOf(it.noteUuid, it.name) } + .filter { it.noteUuid inList uuids } + .groupByTo(HashMap(), { it.note.uuid }, { it.name }) + + return notes.map { note -> + val tags = tagsByUuid[note.uuid] ?: emptyList() + note.toPersistedNote(tags) + } + } } diff --git a/persistance/src/main/kotlin/users/UserRepositoryImpl.kt b/persistance/src/main/kotlin/users/UserRepositoryImpl.kt index b68397c..189eb59 100644 --- a/persistance/src/main/kotlin/users/UserRepositoryImpl.kt +++ b/persistance/src/main/kotlin/users/UserRepositoryImpl.kt @@ -28,4 +28,5 @@ internal class UserRepositoryImpl(private val db: Database) : UserRepository { override fun exists(username: String) = db.users.any { it.username eq username } override fun exists(id: Int) = db.users.any { it.id eq id } override fun delete(id: Int) = db.useTransaction { db.delete(Users) { it.id eq id } == 1 } + override fun findAll() = db.from(Users).select(Users.id).map { it[Users.id]!! } } diff --git a/search/src/main/kotlin/NoteSearcher.kt b/search/src/main/kotlin/NoteSearcher.kt index 5c6126a..8a0c79d 100644 --- a/search/src/main/kotlin/NoteSearcher.kt +++ b/search/src/main/kotlin/NoteSearcher.kt @@ -52,6 +52,21 @@ class NoteSearcher(basePath: Path = Path.of("/tmp", "lucene")) { } } + fun indexNotes(userId: Int, notes: List) { + logger.debug("Indexing notes for user $userId") + + val dir = getDirectory(userId) + val config = IndexWriterConfig(StandardAnalyzer()) + val writer = IndexWriter(dir, config) + val docs = notes.map { it.toDocument() } + + with(writer) { + addDocuments(docs) + commit() + close() + } + } + fun deleteIndex(userId: Int, uuid: UUID) { logger.debug("Deleting indexing $uuid for user $userId") diff --git a/search/src/main/kotlin/SeachModule.kt b/search/src/main/kotlin/SeachModule.kt index 28f7f90..76d0d12 100644 --- a/search/src/main/kotlin/SeachModule.kt +++ b/search/src/main/kotlin/SeachModule.kt @@ -1,7 +1,8 @@ package be.simplenotes.search import org.koin.dsl.module +import java.nio.file.Path val searchModule = module { - single { NoteSearcher() } + single { NoteSearcher(Path.of(".lucene")) } } From 315a01ea18a50eee582ea11a4a568e7a47fd5b06 Mon Sep 17 00:00:00 2001 From: Hubert Van De Walle Date: Wed, 19 Aug 2020 18:19:34 +0200 Subject: [PATCH 4/8] Add search terms parser + tests --- .../main/kotlin/controllers/NoteController.kt | 15 ++++++- app/src/main/kotlin/routes/Router.kt | 1 + .../main/kotlin/utils/SearchTermsParser.kt | 33 ++++++++++++++++ app/src/main/kotlin/views/NoteView.kt | 28 +++++++------ .../kotlin/views/components/NoteListHeader.kt | 19 +++++++++ .../kotlin/utils/SearchTermsParserKtTest.kt | 39 +++++++++++++++++++ .../src/main/kotlin/usecases/NoteService.kt | 3 ++ 7 files changed, 122 insertions(+), 16 deletions(-) create mode 100644 app/src/main/kotlin/utils/SearchTermsParser.kt create mode 100644 app/src/main/kotlin/views/components/NoteListHeader.kt create mode 100644 app/src/test/kotlin/utils/SearchTermsParserKtTest.kt diff --git a/app/src/main/kotlin/controllers/NoteController.kt b/app/src/main/kotlin/controllers/NoteController.kt index f2a3940..ce6d465 100644 --- a/app/src/main/kotlin/controllers/NoteController.kt +++ b/app/src/main/kotlin/controllers/NoteController.kt @@ -2,12 +2,14 @@ package be.simplenotes.app.controllers import be.simplenotes.app.extensions.html import be.simplenotes.app.extensions.redirect +import be.simplenotes.app.utils.parseSearchTerms import be.simplenotes.app.views.NoteView import be.simplenotes.domain.security.JwtPayload import be.simplenotes.domain.usecases.NoteService import be.simplenotes.domain.usecases.markdown.InvalidMeta import be.simplenotes.domain.usecases.markdown.MissingMeta import be.simplenotes.domain.usecases.markdown.ValidationError +import be.simplenotes.search.SearchTerms import org.http4k.core.Method import org.http4k.core.Request import org.http4k.core.Response @@ -34,7 +36,9 @@ class NoteController( val html = when (it) { MissingMeta -> view.noteEditor(jwtPayload, error = "Missing note metadata", textarea = markdownForm) InvalidMeta -> view.noteEditor(jwtPayload, error = "Invalid note metadata", textarea = markdownForm) - is ValidationError -> view.noteEditor(jwtPayload, validationErrors = it.validationErrors, textarea = markdownForm) + is ValidationError -> view.noteEditor(jwtPayload, + validationErrors = it.validationErrors, + textarea = markdownForm) } Response(BAD_REQUEST).html(html) }, @@ -52,6 +56,13 @@ class NoteController( return Response(OK).html(view.notes(jwtPayload, notes, currentPage, pages, deletedCount, tag = tag)) } + fun search(request: Request, jwtPayload: JwtPayload): Response { + val terms = request.searchTerms() + val notes = noteService.search(jwtPayload.userId, terms) + val deletedCount = noteService.countDeleted(jwtPayload.userId) + return Response(OK).html(view.search(jwtPayload, notes, deletedCount)) + } + fun note(request: Request, jwtPayload: JwtPayload): Response { val noteUuid = request.uuidPath() ?: return Response(NOT_FOUND) @@ -121,4 +132,6 @@ class NoteController( null } } + + private fun Request.searchTerms(): SearchTerms = parseSearchTerms(form("search") ?: "") } diff --git a/app/src/main/kotlin/routes/Router.kt b/app/src/main/kotlin/routes/Router.kt index 54a002a..a96db09 100644 --- a/app/src/main/kotlin/routes/Router.kt +++ b/app/src/main/kotlin/routes/Router.kt @@ -48,6 +48,7 @@ class Router( "/settings" bind POST to { protected(it, settingsController::settings) }, "/export" bind POST to { protected(it, settingsController::export) }, "/notes" bind GET to { protected(it, noteController::list) }, + "/notes" bind POST to { protected(it, noteController::search) }, "/notes/new" bind GET to { protected(it, noteController::new) }, "/notes/new" bind POST to { protected(it, noteController::new) }, "/notes/trash" bind GET to { protected(it, noteController::trash) }, diff --git a/app/src/main/kotlin/utils/SearchTermsParser.kt b/app/src/main/kotlin/utils/SearchTermsParser.kt new file mode 100644 index 0000000..0062844 --- /dev/null +++ b/app/src/main/kotlin/utils/SearchTermsParser.kt @@ -0,0 +1,33 @@ +package be.simplenotes.app.utils + +import be.simplenotes.search.SearchTerms + +private val titleRe = """title:['"](?.*?)['"]""".toRegex() +private val outerTitleRe = """(?<title>title:['"].*?['"])""".toRegex() + +private val tagRe = """tag:['"](?<tag>.*?)['"]""".toRegex() +private val outerTagRe = """(?<tag>tag:['"].*?['"])""".toRegex() + +fun parseSearchTerms(input: String): SearchTerms { + val title: String? = titleRe.find(input)?.groups?.get("title")?.value + val tag: String? = tagRe.find(input)?.groups?.get("tag")?.value + var c: String = input + + if (title != null) { + val titleGroup = outerTitleRe.find(input)?.groups?.get("title")?.value + titleGroup?.let { c = c.replace(it, "") } + } + + if (tag != null) { + val tagGroup = outerTagRe.find(input)?.groups?.get("tag")?.value + tagGroup?.let { c = c.replace(it, "") } + } + + val content = c.trim().ifEmpty { null } + + return SearchTerms( + title = title, + tag = tag, + content = content + ) +} diff --git a/app/src/main/kotlin/views/NoteView.kt b/app/src/main/kotlin/views/NoteView.kt index c7d02a1..6e5a6b1 100644 --- a/app/src/main/kotlin/views/NoteView.kt +++ b/app/src/main/kotlin/views/NoteView.kt @@ -55,20 +55,7 @@ class NoteView(staticFileResolver: StaticFileResolver) : View(staticFileResolver tag: String?, ) = renderPage(title = "Notes", jwtPayload = jwtPayload) { div("container mx-auto p-4") { - div("flex justify-between mb-4") { - h1("text-2xl underline") { +"Notes" } - span { - a( - href = "/notes/trash", - classes = "underline font-semibold" - ) { +"Trash ($numberOfDeletedNotes)" } - a( - href = "/notes/new", - classes = "ml-2 btn btn-green" - ) { +"New" } - } - - } + noteListHeader(numberOfDeletedNotes) if (notes.isNotEmpty()) noteTable(notes) else @@ -81,11 +68,22 @@ class NoteView(staticFileResolver: StaticFileResolver) : View(staticFileResolver } } + fun search( + jwtPayload: JwtPayload, + notes: List<PersistedNoteMetadata>, + numberOfDeletedNotes: Int, + ) = renderPage("Notes", jwtPayload = jwtPayload) { + div("container mx-auto p-4") { + noteListHeader(numberOfDeletedNotes) + noteTable(notes) + } + } + fun trash( jwtPayload: JwtPayload, notes: List<PersistedNoteMetadata>, currentPage: Int, - numberOfPages: Int + numberOfPages: Int, ) = renderPage(title = "Notes", jwtPayload = jwtPayload) { div("container mx-auto p-4") { div("flex justify-between mb-4") { diff --git a/app/src/main/kotlin/views/components/NoteListHeader.kt b/app/src/main/kotlin/views/components/NoteListHeader.kt new file mode 100644 index 0000000..a315060 --- /dev/null +++ b/app/src/main/kotlin/views/components/NoteListHeader.kt @@ -0,0 +1,19 @@ +package be.simplenotes.app.views.components + +import kotlinx.html.* + +fun DIV.noteListHeader(numberOfDeletedNotes: Int) { + div("flex justify-between mb-4") { + h1("text-2xl underline") { +"Notes" } + span { + a( + href = "/notes/trash", + classes = "underline font-semibold" + ) { +"Trash ($numberOfDeletedNotes)" } + a( + href = "/notes/new", + classes = "ml-2 btn btn-green" + ) { +"New" } + } + } +} diff --git a/app/src/test/kotlin/utils/SearchTermsParserKtTest.kt b/app/src/test/kotlin/utils/SearchTermsParserKtTest.kt new file mode 100644 index 0000000..39f996a --- /dev/null +++ b/app/src/test/kotlin/utils/SearchTermsParserKtTest.kt @@ -0,0 +1,39 @@ +package be.simplenotes.app.utils + +import be.simplenotes.search.SearchTerms +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import java.util.stream.Stream + +internal class SearchTermsParserKtTest { + + private fun createResult( + input: String, + title: String? = null, + tag: String? = null, + content: String? = null, + ): Pair<String, SearchTerms> = input to SearchTerms(title, tag, content) + + @Suppress("Unused") + private fun results() = Stream.of( + createResult("title:'example'", title = "example"), + createResult("title:'example with words'", title = "example with words"), + createResult("title:'example with words'", title = "example with words"), + createResult("""title:"double quotes"""", title = "double quotes"), + createResult("title:'example' something else", title = "example", content = "something else"), + createResult("tag:'example'", tag = "example"), + createResult("tag:'example' title:'other'", title = "other", tag = "example"), + createResult("blah blah tag:'example' title:'other'", title = "other", tag = "example", content = "blah blah"), + createResult("tag:'example' middle title:'other'", title = "other", tag = "example", content = "middle"), + createResult("tag:'example' title:'other' end", title = "other", tag = "example", content = "end"), + createResult("tag:'example abc' title:'other with words' this is the end ", title = "other with words", tag = "example abc", content = "this is the end"), + ) + + @ParameterizedTest + @MethodSource("results") + fun `valid search parser`(case: Pair<String, SearchTerms>) { + assertThat(parseSearchTerms(case.first)).isEqualTo(case.second) + } + +} diff --git a/domain/src/main/kotlin/usecases/NoteService.kt b/domain/src/main/kotlin/usecases/NoteService.kt index d28d9ef..d7fdabd 100644 --- a/domain/src/main/kotlin/usecases/NoteService.kt +++ b/domain/src/main/kotlin/usecases/NoteService.kt @@ -11,6 +11,7 @@ import be.simplenotes.domain.usecases.markdown.MarkdownParsingError import be.simplenotes.domain.usecases.repositories.NoteRepository import be.simplenotes.domain.usecases.repositories.UserRepository import be.simplenotes.search.NoteSearcher +import be.simplenotes.search.SearchTerms import java.util.* class NoteService( @@ -83,6 +84,8 @@ class NoteService( searcher.indexNotes(id, notes) } } + + fun search(userId: Int, searchTerms: SearchTerms) = searcher.search(userId, searchTerms) } data class PaginatedNotes(val pages: Int, val notes: List<PersistedNoteMetadata>) From 68109f8666fb13d4b61fe3ffd1f6f7442f8fe874 Mon Sep 17 00:00:00 2001 From: Hubert Van De Walle <hubv@protonmail.com> Date: Wed, 19 Aug 2020 18:47:19 +0200 Subject: [PATCH 5/8] Drop indexes + view --- app/src/main/kotlin/SimpleNotes.kt | 1 + .../kotlin/views/components/NoteListHeader.kt | 12 ++++++++ .../src/main/kotlin/usecases/NoteService.kt | 2 ++ persistance/src/main/resources/logback.xml | 4 +-- search/src/main/kotlin/NoteSearcher.kt | 28 ++++-------------- search/src/main/kotlin/utils/PathUtils.kt | 29 +++++++++++++++++++ 6 files changed, 52 insertions(+), 24 deletions(-) create mode 100644 search/src/main/kotlin/utils/PathUtils.kt diff --git a/app/src/main/kotlin/SimpleNotes.kt b/app/src/main/kotlin/SimpleNotes.kt index 8bd5060..b73389b 100644 --- a/app/src/main/kotlin/SimpleNotes.kt +++ b/app/src/main/kotlin/SimpleNotes.kt @@ -54,6 +54,7 @@ fun main() { migrations.migrate() val noteService = koin.get<NoteService>() + noteService.dropAllIndexes() noteService.indexAll() koin.get<Server>().start() diff --git a/app/src/main/kotlin/views/components/NoteListHeader.kt b/app/src/main/kotlin/views/components/NoteListHeader.kt index a315060..9f7aefc 100644 --- a/app/src/main/kotlin/views/components/NoteListHeader.kt +++ b/app/src/main/kotlin/views/components/NoteListHeader.kt @@ -1,6 +1,8 @@ package be.simplenotes.app.views.components import kotlinx.html.* +import kotlinx.html.ButtonType.submit +import kotlinx.html.FormMethod.post fun DIV.noteListHeader(numberOfDeletedNotes: Int) { div("flex justify-between mb-4") { @@ -16,4 +18,14 @@ fun DIV.noteListHeader(numberOfDeletedNotes: Int) { ) { +"New" } } } + form(method = post, classes = "mb-4") { + val colors = "bg-gray-800 border-gray-700 focus:border-teal-500 text-white" + input( + name = "search", + classes = "$colors rounded w-3/4 border appearance-none focus:outline-none text-base p-2" + ) + button(type = submit, classes = "btn btn-green w-1/4") { + +"search" + } + } } diff --git a/domain/src/main/kotlin/usecases/NoteService.kt b/domain/src/main/kotlin/usecases/NoteService.kt index d7fdabd..f03fa37 100644 --- a/domain/src/main/kotlin/usecases/NoteService.kt +++ b/domain/src/main/kotlin/usecases/NoteService.kt @@ -86,6 +86,8 @@ class NoteService( } fun search(userId: Int, searchTerms: SearchTerms) = searcher.search(userId, searchTerms) + + fun dropAllIndexes() = searcher.dropAll() } data class PaginatedNotes(val pages: Int, val notes: List<PersistedNoteMetadata>) diff --git a/persistance/src/main/resources/logback.xml b/persistance/src/main/resources/logback.xml index 5024926..d94bc53 100644 --- a/persistance/src/main/resources/logback.xml +++ b/persistance/src/main/resources/logback.xml @@ -6,11 +6,11 @@ </pattern> </encoder> </appender> - <root level="INFO"> + <root level="DEBUG"> <appender-ref ref="STDOUT"/> </root> <logger name="org.eclipse.jetty" level="INFO"/> - <logger name="me.liuwj.ktorm.database" level="DEBUG"/> + <logger name="me.liuwj.ktorm.database" level="INFO"/> <logger name="com.zaxxer.hikari" level="INFO"/> <logger name="org.flywaydb.core" level="INFO"/> </configuration> diff --git a/search/src/main/kotlin/NoteSearcher.kt b/search/src/main/kotlin/NoteSearcher.kt index 8a0c79d..27d59f4 100644 --- a/search/src/main/kotlin/NoteSearcher.kt +++ b/search/src/main/kotlin/NoteSearcher.kt @@ -2,6 +2,7 @@ package be.simplenotes.search import be.simplenotes.domain.model.PersistedNote import be.simplenotes.domain.model.PersistedNoteMetadata +import be.simplenotes.search.utils.rmdir import org.apache.lucene.analysis.standard.StandardAnalyzer import org.apache.lucene.index.* import org.apache.lucene.search.* @@ -68,7 +69,7 @@ class NoteSearcher(basePath: Path = Path.of("/tmp", "lucene")) { } fun deleteIndex(userId: Int, uuid: UUID) { - logger.debug("Deleting indexing $uuid for user $userId") + logger.debug("Deleting index $uuid for user $userId") val dir = getDirectory(userId) val config = IndexWriterConfig(StandardAnalyzer()) @@ -108,31 +109,14 @@ class NoteSearcher(basePath: Path = Path.of("/tmp", "lucene")) { } val query = builder.build() - logger.debug("Searching: $query") val topDocs = searcher.search(query, 10) + logger.debug("Searching: `$query` results: ${topDocs.totalHits.value}") return topDocs.toResults(searcher) } - fun dropIndex(userId: Int) { - val index = File(baseFile, userId.toString()).toPath() - try { - Files.walkFileTree( - index, - object : SimpleFileVisitor<Path>() { - override fun visitFile(file: Path, attrs: BasicFileAttributes?): FileVisitResult { - Files.delete(file) - return FileVisitResult.CONTINUE - } + fun dropIndex(userId: Int) = rmdir(File(baseFile, userId.toString()).toPath()) + + fun dropAll() = rmdir(baseFile.toPath()) - override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult { - Files.delete(dir) - return FileVisitResult.CONTINUE - } - } - ) - } catch (e: IOException) { - // This is fine - } - } } diff --git a/search/src/main/kotlin/utils/PathUtils.kt b/search/src/main/kotlin/utils/PathUtils.kt new file mode 100644 index 0000000..fd5966e --- /dev/null +++ b/search/src/main/kotlin/utils/PathUtils.kt @@ -0,0 +1,29 @@ +package be.simplenotes.search.utils + +import java.io.IOException +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes + +internal fun rmdir(path: Path) { + try { + Files.walkFileTree( + path, + object : SimpleFileVisitor<Path>() { + override fun visitFile(file: Path, attrs: BasicFileAttributes?): FileVisitResult { + Files.delete(file) + return FileVisitResult.CONTINUE + } + + override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult { + Files.delete(dir) + return FileVisitResult.CONTINUE + } + } + ) + } catch (e: IOException) { + // This is fine + } +} From 08c804ccb54b5bec69b3785699ec173b9081018d Mon Sep 17 00:00:00 2001 From: Hubert Van De Walle <hubv@protonmail.com> Date: Wed, 19 Aug 2020 19:02:15 +0200 Subject: [PATCH 6/8] Fix circular dependency --- app/pom.xml | 5 ++++ .../main/kotlin/controllers/NoteController.kt | 2 +- .../main/kotlin/utils/SearchTermsParser.kt | 2 +- .../kotlin/utils/SearchTermsParserKtTest.kt | 2 +- domain/pom.xml | 5 ---- .../src/main/kotlin/usecases/NoteService.kt | 4 +-- .../kotlin/usecases/search/SearchUseCase.kt | 17 +++++++++++++ .../{NoteSearcher.kt => NoteSearcherImpl.kt} | 25 ++++++++----------- search/src/main/kotlin/SeachModule.kt | 3 ++- ...earcherTest.kt => NoteSearcherImplTest.kt} | 5 ++-- 10 files changed, 42 insertions(+), 28 deletions(-) create mode 100644 domain/src/main/kotlin/usecases/search/SearchUseCase.kt rename search/src/main/kotlin/{NoteSearcher.kt => NoteSearcherImpl.kt} (80%) rename search/src/test/kotlin/{NoteSearcherTest.kt => NoteSearcherImplTest.kt} (97%) diff --git a/app/pom.xml b/app/pom.xml index c578cec..062d664 100644 --- a/app/pom.xml +++ b/app/pom.xml @@ -14,6 +14,11 @@ <artifactId>persistance</artifactId> <version>1.0-SNAPSHOT</version> </dependency> + <dependency> + <groupId>be.simplenotes</groupId> + <artifactId>search</artifactId> + <version>1.0-SNAPSHOT</version> + </dependency> <dependency> <groupId>be.simplenotes</groupId> <artifactId>domain</artifactId> diff --git a/app/src/main/kotlin/controllers/NoteController.kt b/app/src/main/kotlin/controllers/NoteController.kt index ce6d465..8d9abc0 100644 --- a/app/src/main/kotlin/controllers/NoteController.kt +++ b/app/src/main/kotlin/controllers/NoteController.kt @@ -9,7 +9,7 @@ import be.simplenotes.domain.usecases.NoteService import be.simplenotes.domain.usecases.markdown.InvalidMeta import be.simplenotes.domain.usecases.markdown.MissingMeta import be.simplenotes.domain.usecases.markdown.ValidationError -import be.simplenotes.search.SearchTerms +import be.simplenotes.domain.usecases.search.SearchTerms import org.http4k.core.Method import org.http4k.core.Request import org.http4k.core.Response diff --git a/app/src/main/kotlin/utils/SearchTermsParser.kt b/app/src/main/kotlin/utils/SearchTermsParser.kt index 0062844..f7dc499 100644 --- a/app/src/main/kotlin/utils/SearchTermsParser.kt +++ b/app/src/main/kotlin/utils/SearchTermsParser.kt @@ -1,6 +1,6 @@ package be.simplenotes.app.utils -import be.simplenotes.search.SearchTerms +import be.simplenotes.domain.usecases.search.SearchTerms private val titleRe = """title:['"](?<title>.*?)['"]""".toRegex() private val outerTitleRe = """(?<title>title:['"].*?['"])""".toRegex() diff --git a/app/src/test/kotlin/utils/SearchTermsParserKtTest.kt b/app/src/test/kotlin/utils/SearchTermsParserKtTest.kt index 39f996a..a98334d 100644 --- a/app/src/test/kotlin/utils/SearchTermsParserKtTest.kt +++ b/app/src/test/kotlin/utils/SearchTermsParserKtTest.kt @@ -1,6 +1,6 @@ package be.simplenotes.app.utils -import be.simplenotes.search.SearchTerms +import be.simplenotes.domain.usecases.search.SearchTerms import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource diff --git a/domain/pom.xml b/domain/pom.xml index 08682fe..82f582d 100644 --- a/domain/pom.xml +++ b/domain/pom.xml @@ -14,11 +14,6 @@ <artifactId>shared</artifactId> <version>1.0-SNAPSHOT</version> </dependency> - <dependency> - <groupId>be.simplenotes</groupId> - <artifactId>search</artifactId> - <version>1.0-SNAPSHOT</version> - </dependency> <dependency> <groupId>be.simplenotes</groupId> <artifactId>shared</artifactId> diff --git a/domain/src/main/kotlin/usecases/NoteService.kt b/domain/src/main/kotlin/usecases/NoteService.kt index f03fa37..ef672f0 100644 --- a/domain/src/main/kotlin/usecases/NoteService.kt +++ b/domain/src/main/kotlin/usecases/NoteService.kt @@ -10,8 +10,8 @@ import be.simplenotes.domain.usecases.markdown.MarkdownConverter import be.simplenotes.domain.usecases.markdown.MarkdownParsingError import be.simplenotes.domain.usecases.repositories.NoteRepository import be.simplenotes.domain.usecases.repositories.UserRepository -import be.simplenotes.search.NoteSearcher -import be.simplenotes.search.SearchTerms +import be.simplenotes.domain.usecases.search.NoteSearcher +import be.simplenotes.domain.usecases.search.SearchTerms import java.util.* class NoteService( diff --git a/domain/src/main/kotlin/usecases/search/SearchUseCase.kt b/domain/src/main/kotlin/usecases/search/SearchUseCase.kt new file mode 100644 index 0000000..d2d406c --- /dev/null +++ b/domain/src/main/kotlin/usecases/search/SearchUseCase.kt @@ -0,0 +1,17 @@ +package be.simplenotes.domain.usecases.search + +import be.simplenotes.domain.model.PersistedNote +import be.simplenotes.domain.model.PersistedNoteMetadata +import java.util.* + +data class SearchTerms(val title: String?, val tag: String?, val content: String?) + +interface NoteSearcher { + fun indexNote(userId: Int, note: PersistedNote) + fun indexNotes(userId: Int, notes: List<PersistedNote>) + fun deleteIndex(userId: Int, uuid: UUID) + fun updateIndex(userId: Int, note: PersistedNote) + fun search(userId: Int, terms: SearchTerms): List<PersistedNoteMetadata> + fun dropIndex(userId: Int) + fun dropAll() +} diff --git a/search/src/main/kotlin/NoteSearcher.kt b/search/src/main/kotlin/NoteSearcherImpl.kt similarity index 80% rename from search/src/main/kotlin/NoteSearcher.kt rename to search/src/main/kotlin/NoteSearcherImpl.kt index 27d59f4..98f9237 100644 --- a/search/src/main/kotlin/NoteSearcher.kt +++ b/search/src/main/kotlin/NoteSearcherImpl.kt @@ -2,6 +2,8 @@ package be.simplenotes.search import be.simplenotes.domain.model.PersistedNote import be.simplenotes.domain.model.PersistedNoteMetadata +import be.simplenotes.domain.usecases.search.NoteSearcher +import be.simplenotes.domain.usecases.search.SearchTerms import be.simplenotes.search.utils.rmdir import org.apache.lucene.analysis.standard.StandardAnalyzer import org.apache.lucene.index.* @@ -10,17 +12,10 @@ import org.apache.lucene.store.Directory import org.apache.lucene.store.FSDirectory import org.slf4j.LoggerFactory import java.io.File -import java.io.IOException -import java.nio.file.FileVisitResult -import java.nio.file.Files import java.nio.file.Path -import java.nio.file.SimpleFileVisitor -import java.nio.file.attribute.BasicFileAttributes import java.util.* -data class SearchTerms(val title: String?, val tag: String?, val content: String?) - -class NoteSearcher(basePath: Path = Path.of("/tmp", "lucene")) { +class NoteSearcherImpl(basePath: Path = Path.of("/tmp", "lucene")) : NoteSearcher { private val baseFile = basePath.toFile() private val logger = LoggerFactory.getLogger(javaClass) @@ -38,7 +33,7 @@ class NoteSearcher(basePath: Path = Path.of("/tmp", "lucene")) { } // endregion - fun indexNote(userId: Int, note: PersistedNote) { + override fun indexNote(userId: Int, note: PersistedNote) { logger.debug("Indexing note ${note.uuid} for user $userId") val dir = getDirectory(userId) @@ -53,7 +48,7 @@ class NoteSearcher(basePath: Path = Path.of("/tmp", "lucene")) { } } - fun indexNotes(userId: Int, notes: List<PersistedNote>) { + override fun indexNotes(userId: Int, notes: List<PersistedNote>) { logger.debug("Indexing notes for user $userId") val dir = getDirectory(userId) @@ -68,7 +63,7 @@ class NoteSearcher(basePath: Path = Path.of("/tmp", "lucene")) { } } - fun deleteIndex(userId: Int, uuid: UUID) { + override fun deleteIndex(userId: Int, uuid: UUID) { logger.debug("Deleting index $uuid for user $userId") val dir = getDirectory(userId) @@ -82,13 +77,13 @@ class NoteSearcher(basePath: Path = Path.of("/tmp", "lucene")) { } } - fun updateIndex(userId: Int, note: PersistedNote) { + override fun updateIndex(userId: Int, note: PersistedNote) { logger.debug("Updating note ${note.uuid} for user $userId") deleteIndex(userId, note.uuid) indexNote(userId, note) } - fun search(userId: Int, terms: SearchTerms): List<PersistedNoteMetadata> { + override fun search(userId: Int, terms: SearchTerms): List<PersistedNoteMetadata> { val searcher = getIndexSearcher(userId) val builder = BooleanQuery.Builder() @@ -115,8 +110,8 @@ class NoteSearcher(basePath: Path = Path.of("/tmp", "lucene")) { return topDocs.toResults(searcher) } - fun dropIndex(userId: Int) = rmdir(File(baseFile, userId.toString()).toPath()) + override fun dropIndex(userId: Int) = rmdir(File(baseFile, userId.toString()).toPath()) - fun dropAll() = rmdir(baseFile.toPath()) + override fun dropAll() = rmdir(baseFile.toPath()) } diff --git a/search/src/main/kotlin/SeachModule.kt b/search/src/main/kotlin/SeachModule.kt index 76d0d12..d86f399 100644 --- a/search/src/main/kotlin/SeachModule.kt +++ b/search/src/main/kotlin/SeachModule.kt @@ -1,8 +1,9 @@ package be.simplenotes.search +import be.simplenotes.domain.usecases.search.NoteSearcher import org.koin.dsl.module import java.nio.file.Path val searchModule = module { - single { NoteSearcher(Path.of(".lucene")) } + single<NoteSearcher> { NoteSearcherImpl(Path.of(".lucene")) } } diff --git a/search/src/test/kotlin/NoteSearcherTest.kt b/search/src/test/kotlin/NoteSearcherImplTest.kt similarity index 97% rename from search/src/test/kotlin/NoteSearcherTest.kt rename to search/src/test/kotlin/NoteSearcherImplTest.kt index 357f1d5..b6a5eae 100644 --- a/search/src/test/kotlin/NoteSearcherTest.kt +++ b/search/src/test/kotlin/NoteSearcherImplTest.kt @@ -3,6 +3,7 @@ package be.simplenotes.search import be.simplenotes.domain.model.NoteMetadata import be.simplenotes.domain.model.PersistedNote import be.simplenotes.domain.model.PersistedNoteMetadata +import be.simplenotes.domain.usecases.search.SearchTerms import org.assertj.core.api.Assertions.assertThat import org.intellij.lang.annotations.Language import org.junit.jupiter.api.AfterAll @@ -13,10 +14,10 @@ import java.time.LocalDateTime import java.util.* @ResourceLock("lucene") -internal class NoteSearcherTest { +internal class NoteSearcherImplTest { // region setup - private val searcher = NoteSearcher() + private val searcher = NoteSearcherImpl() private fun index( title: String, From 1432fbb395b6859e5b1ff81ba59458fe0b9bc687 Mon Sep 17 00:00:00 2001 From: Hubert Van De Walle <hubv@protonmail.com> Date: Wed, 19 Aug 2020 19:17:06 +0200 Subject: [PATCH 7/8] Update docker + pom --- Dockerfile | 2 ++ app/pom.xml | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6f30e6a..3598e52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ COPY app/pom.xml app/pom.xml COPY domain/pom.xml domain/pom.xml COPY persistance/pom.xml persistance/pom.xml COPY shared/pom.xml shared/pom.xml +COPY search/pom.xml search/pom.xml RUN mvn verify clean --fail-never @@ -15,6 +16,7 @@ COPY app/src app/src COPY domain/src domain/src COPY persistance/src persistance/src COPY shared/src shared/src +COPY search/src search/src RUN mvn -Dstyle.color=always package diff --git a/app/pom.xml b/app/pom.xml index 062d664..4078e4d 100644 --- a/app/pom.xml +++ b/app/pom.xml @@ -109,6 +109,12 @@ <include>**</include> </includes> </filter> + <filter> + <artifact>org.apache.lucene:*</artifact> + <includes> + <include>**</include> + </includes> + </filter> <filter> <artifact>*:*</artifact> <excludes> From a44019900660979e9d33b16a2dd58bd7209ebfcf Mon Sep 17 00:00:00 2001 From: Hubert Van De Walle <hubv@protonmail.com> Date: Wed, 19 Aug 2020 19:21:54 +0200 Subject: [PATCH 8/8] Fix unsupported method on jdk backend --- app/src/main/kotlin/utils/SearchTermsParser.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/main/kotlin/utils/SearchTermsParser.kt b/app/src/main/kotlin/utils/SearchTermsParser.kt index f7dc499..733f43e 100644 --- a/app/src/main/kotlin/utils/SearchTermsParser.kt +++ b/app/src/main/kotlin/utils/SearchTermsParser.kt @@ -9,17 +9,17 @@ private val tagRe = """tag:['"](?<tag>.*?)['"]""".toRegex() private val outerTagRe = """(?<tag>tag:['"].*?['"])""".toRegex() fun parseSearchTerms(input: String): SearchTerms { - val title: String? = titleRe.find(input)?.groups?.get("title")?.value - val tag: String? = tagRe.find(input)?.groups?.get("tag")?.value + val title: String? = titleRe.find(input)?.groups?.get(1)?.value + val tag: String? = tagRe.find(input)?.groups?.get(1)?.value var c: String = input if (title != null) { - val titleGroup = outerTitleRe.find(input)?.groups?.get("title")?.value + val titleGroup = outerTitleRe.find(input)?.groups?.get(1)?.value titleGroup?.let { c = c.replace(it, "") } } if (tag != null) { - val tagGroup = outerTagRe.find(input)?.groups?.get("tag")?.value + val tagGroup = outerTagRe.find(input)?.groups?.get(1)?.value tagGroup?.let { c = c.replace(it, "") } }