diff --git a/.gitignore b/.gitignore index baa62e3..fa745c3 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,6 @@ app/src/main/resources/static/styles* # h2 db *.db + +# lucene index +.lucene/ diff --git a/Dockerfile b/Dockerfile index 6f30e6a..3598e52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ COPY app/pom.xml app/pom.xml COPY domain/pom.xml domain/pom.xml COPY persistance/pom.xml persistance/pom.xml COPY shared/pom.xml shared/pom.xml +COPY search/pom.xml search/pom.xml RUN mvn verify clean --fail-never @@ -15,6 +16,7 @@ COPY app/src app/src COPY domain/src domain/src COPY persistance/src persistance/src COPY shared/src shared/src +COPY search/src search/src RUN mvn -Dstyle.color=always package diff --git a/app/pom.xml b/app/pom.xml index c578cec..4078e4d 100644 --- a/app/pom.xml +++ b/app/pom.xml @@ -14,6 +14,11 @@ persistance 1.0-SNAPSHOT + + be.simplenotes + search + 1.0-SNAPSHOT + be.simplenotes domain @@ -104,6 +109,12 @@ ** + + org.apache.lucene:* + + ** + + *:* diff --git a/app/src/main/kotlin/SimpleNotes.kt b/app/src/main/kotlin/SimpleNotes.kt index 984fefe..b73389b 100644 --- a/app/src/main/kotlin/SimpleNotes.kt +++ b/app/src/main/kotlin/SimpleNotes.kt @@ -14,8 +14,10 @@ import be.simplenotes.app.views.NoteView import be.simplenotes.app.views.SettingView import be.simplenotes.app.views.UserView import be.simplenotes.domain.domainModule +import be.simplenotes.domain.usecases.NoteService import be.simplenotes.persistance.DbMigrations import be.simplenotes.persistance.persistanceModule +import be.simplenotes.search.searchModule import be.simplenotes.shared.config.DataSourceConfig import be.simplenotes.shared.config.JwtConfig import org.http4k.core.RequestContexts @@ -36,6 +38,7 @@ fun main() { baseModule, noteModule, settingsModule, + searchModule, ) }.koin @@ -50,6 +53,10 @@ fun main() { val migrations = koin.get() migrations.migrate() + val noteService = koin.get() + noteService.dropAllIndexes() + noteService.indexAll() + koin.get().start() } diff --git a/app/src/main/kotlin/controllers/NoteController.kt b/app/src/main/kotlin/controllers/NoteController.kt index f2a3940..8d9abc0 100644 --- a/app/src/main/kotlin/controllers/NoteController.kt +++ b/app/src/main/kotlin/controllers/NoteController.kt @@ -2,12 +2,14 @@ package be.simplenotes.app.controllers import be.simplenotes.app.extensions.html import be.simplenotes.app.extensions.redirect +import be.simplenotes.app.utils.parseSearchTerms import be.simplenotes.app.views.NoteView import be.simplenotes.domain.security.JwtPayload import be.simplenotes.domain.usecases.NoteService import be.simplenotes.domain.usecases.markdown.InvalidMeta import be.simplenotes.domain.usecases.markdown.MissingMeta import be.simplenotes.domain.usecases.markdown.ValidationError +import be.simplenotes.domain.usecases.search.SearchTerms import org.http4k.core.Method import org.http4k.core.Request import org.http4k.core.Response @@ -34,7 +36,9 @@ class NoteController( val html = when (it) { MissingMeta -> view.noteEditor(jwtPayload, error = "Missing note metadata", textarea = markdownForm) InvalidMeta -> view.noteEditor(jwtPayload, error = "Invalid note metadata", textarea = markdownForm) - is ValidationError -> view.noteEditor(jwtPayload, validationErrors = it.validationErrors, textarea = markdownForm) + is ValidationError -> view.noteEditor(jwtPayload, + validationErrors = it.validationErrors, + textarea = markdownForm) } Response(BAD_REQUEST).html(html) }, @@ -52,6 +56,13 @@ class NoteController( return Response(OK).html(view.notes(jwtPayload, notes, currentPage, pages, deletedCount, tag = tag)) } + fun search(request: Request, jwtPayload: JwtPayload): Response { + val terms = request.searchTerms() + val notes = noteService.search(jwtPayload.userId, terms) + val deletedCount = noteService.countDeleted(jwtPayload.userId) + return Response(OK).html(view.search(jwtPayload, notes, deletedCount)) + } + fun note(request: Request, jwtPayload: JwtPayload): Response { val noteUuid = request.uuidPath() ?: return Response(NOT_FOUND) @@ -121,4 +132,6 @@ class NoteController( null } } + + private fun Request.searchTerms(): SearchTerms = parseSearchTerms(form("search") ?: "") } diff --git a/app/src/main/kotlin/routes/Router.kt b/app/src/main/kotlin/routes/Router.kt index 54a002a..a96db09 100644 --- a/app/src/main/kotlin/routes/Router.kt +++ b/app/src/main/kotlin/routes/Router.kt @@ -48,6 +48,7 @@ class Router( "/settings" bind POST to { protected(it, settingsController::settings) }, "/export" bind POST to { protected(it, settingsController::export) }, "/notes" bind GET to { protected(it, noteController::list) }, + "/notes" bind POST to { protected(it, noteController::search) }, "/notes/new" bind GET to { protected(it, noteController::new) }, "/notes/new" bind POST to { protected(it, noteController::new) }, "/notes/trash" bind GET to { protected(it, noteController::trash) }, diff --git a/app/src/main/kotlin/utils/SearchTermsParser.kt b/app/src/main/kotlin/utils/SearchTermsParser.kt new file mode 100644 index 0000000..733f43e --- /dev/null +++ b/app/src/main/kotlin/utils/SearchTermsParser.kt @@ -0,0 +1,33 @@ +package be.simplenotes.app.utils + +import be.simplenotes.domain.usecases.search.SearchTerms + +private val titleRe = """title:['"](?.*?)['"]""".toRegex() +private val outerTitleRe = """(?<title>title:['"].*?['"])""".toRegex() + +private val tagRe = """tag:['"](?<tag>.*?)['"]""".toRegex() +private val outerTagRe = """(?<tag>tag:['"].*?['"])""".toRegex() + +fun parseSearchTerms(input: String): SearchTerms { + val title: String? = titleRe.find(input)?.groups?.get(1)?.value + val tag: String? = tagRe.find(input)?.groups?.get(1)?.value + var c: String = input + + if (title != null) { + val titleGroup = outerTitleRe.find(input)?.groups?.get(1)?.value + titleGroup?.let { c = c.replace(it, "") } + } + + if (tag != null) { + val tagGroup = outerTagRe.find(input)?.groups?.get(1)?.value + tagGroup?.let { c = c.replace(it, "") } + } + + val content = c.trim().ifEmpty { null } + + return SearchTerms( + title = title, + tag = tag, + content = content + ) +} diff --git a/app/src/main/kotlin/views/NoteView.kt b/app/src/main/kotlin/views/NoteView.kt index c7d02a1..6e5a6b1 100644 --- a/app/src/main/kotlin/views/NoteView.kt +++ b/app/src/main/kotlin/views/NoteView.kt @@ -55,20 +55,7 @@ class NoteView(staticFileResolver: StaticFileResolver) : View(staticFileResolver tag: String?, ) = renderPage(title = "Notes", jwtPayload = jwtPayload) { div("container mx-auto p-4") { - div("flex justify-between mb-4") { - h1("text-2xl underline") { +"Notes" } - span { - a( - href = "/notes/trash", - classes = "underline font-semibold" - ) { +"Trash ($numberOfDeletedNotes)" } - a( - href = "/notes/new", - classes = "ml-2 btn btn-green" - ) { +"New" } - } - - } + noteListHeader(numberOfDeletedNotes) if (notes.isNotEmpty()) noteTable(notes) else @@ -81,11 +68,22 @@ class NoteView(staticFileResolver: StaticFileResolver) : View(staticFileResolver } } + fun search( + jwtPayload: JwtPayload, + notes: List<PersistedNoteMetadata>, + numberOfDeletedNotes: Int, + ) = renderPage("Notes", jwtPayload = jwtPayload) { + div("container mx-auto p-4") { + noteListHeader(numberOfDeletedNotes) + noteTable(notes) + } + } + fun trash( jwtPayload: JwtPayload, notes: List<PersistedNoteMetadata>, currentPage: Int, - numberOfPages: Int + numberOfPages: Int, ) = renderPage(title = "Notes", jwtPayload = jwtPayload) { div("container mx-auto p-4") { div("flex justify-between mb-4") { diff --git a/app/src/main/kotlin/views/components/NoteListHeader.kt b/app/src/main/kotlin/views/components/NoteListHeader.kt new file mode 100644 index 0000000..9f7aefc --- /dev/null +++ b/app/src/main/kotlin/views/components/NoteListHeader.kt @@ -0,0 +1,31 @@ +package be.simplenotes.app.views.components + +import kotlinx.html.* +import kotlinx.html.ButtonType.submit +import kotlinx.html.FormMethod.post + +fun DIV.noteListHeader(numberOfDeletedNotes: Int) { + div("flex justify-between mb-4") { + h1("text-2xl underline") { +"Notes" } + span { + a( + href = "/notes/trash", + classes = "underline font-semibold" + ) { +"Trash ($numberOfDeletedNotes)" } + a( + href = "/notes/new", + classes = "ml-2 btn btn-green" + ) { +"New" } + } + } + form(method = post, classes = "mb-4") { + val colors = "bg-gray-800 border-gray-700 focus:border-teal-500 text-white" + input( + name = "search", + classes = "$colors rounded w-3/4 border appearance-none focus:outline-none text-base p-2" + ) + button(type = submit, classes = "btn btn-green w-1/4") { + +"search" + } + } +} diff --git a/app/src/test/kotlin/utils/SearchTermsParserKtTest.kt b/app/src/test/kotlin/utils/SearchTermsParserKtTest.kt new file mode 100644 index 0000000..a98334d --- /dev/null +++ b/app/src/test/kotlin/utils/SearchTermsParserKtTest.kt @@ -0,0 +1,39 @@ +package be.simplenotes.app.utils + +import be.simplenotes.domain.usecases.search.SearchTerms +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import java.util.stream.Stream + +internal class SearchTermsParserKtTest { + + private fun createResult( + input: String, + title: String? = null, + tag: String? = null, + content: String? = null, + ): Pair<String, SearchTerms> = input to SearchTerms(title, tag, content) + + @Suppress("Unused") + private fun results() = Stream.of( + createResult("title:'example'", title = "example"), + createResult("title:'example with words'", title = "example with words"), + createResult("title:'example with words'", title = "example with words"), + createResult("""title:"double quotes"""", title = "double quotes"), + createResult("title:'example' something else", title = "example", content = "something else"), + createResult("tag:'example'", tag = "example"), + createResult("tag:'example' title:'other'", title = "other", tag = "example"), + createResult("blah blah tag:'example' title:'other'", title = "other", tag = "example", content = "blah blah"), + createResult("tag:'example' middle title:'other'", title = "other", tag = "example", content = "middle"), + createResult("tag:'example' title:'other' end", title = "other", tag = "example", content = "end"), + createResult("tag:'example abc' title:'other with words' this is the end ", title = "other with words", tag = "example abc", content = "this is the end"), + ) + + @ParameterizedTest + @MethodSource("results") + fun `valid search parser`(case: Pair<String, SearchTerms>) { + assertThat(parseSearchTerms(case.first)).isEqualTo(case.second) + } + +} diff --git a/domain/src/main/kotlin/DomainModule.kt b/domain/src/main/kotlin/DomainModule.kt index 2af4176..6d4de57 100644 --- a/domain/src/main/kotlin/DomainModule.kt +++ b/domain/src/main/kotlin/DomainModule.kt @@ -26,7 +26,7 @@ val domainModule = module { single<PasswordHash> { BcryptPasswordHash() } single { SimpleJwt(get()) } single { JwtPayloadExtractor(get()) } - single { NoteService(get(), get()) } + single { NoteService(get(), get(), get(), get()) } single<MarkdownConverter> { MarkdownConverterImpl() } single<ExportUseCase> { ExportUseCaseImpl(get()) } } diff --git a/domain/src/main/kotlin/usecases/NoteService.kt b/domain/src/main/kotlin/usecases/NoteService.kt index 196db02..ef672f0 100644 --- a/domain/src/main/kotlin/usecases/NoteService.kt +++ b/domain/src/main/kotlin/usecases/NoteService.kt @@ -1,6 +1,7 @@ package be.simplenotes.domain.usecases import arrow.core.Either +import arrow.core.extensions.fx import be.simplenotes.domain.model.Note import be.simplenotes.domain.model.PersistedNote import be.simplenotes.domain.model.PersistedNoteMetadata @@ -8,33 +9,44 @@ import be.simplenotes.domain.security.HtmlSanitizer import be.simplenotes.domain.usecases.markdown.MarkdownConverter import be.simplenotes.domain.usecases.markdown.MarkdownParsingError import be.simplenotes.domain.usecases.repositories.NoteRepository +import be.simplenotes.domain.usecases.repositories.UserRepository +import be.simplenotes.domain.usecases.search.NoteSearcher +import be.simplenotes.domain.usecases.search.SearchTerms import java.util.* class NoteService( private val markdownConverter: MarkdownConverter, private val noteRepository: NoteRepository, + private val userRepository: UserRepository, + private val searcher: NoteSearcher, ) { - fun create(userId: Int, markdownText: String): Either<MarkdownParsingError, PersistedNote> = - markdownConverter - .renderDocument(markdownText) + fun create(userId: Int, markdownText: String) = Either.fx<MarkdownParsingError, PersistedNote> { + val persistedNote = !markdownConverter.renderDocument(markdownText) .map { it.copy(html = HtmlSanitizer.sanitize(it.html)) } .map { Note(it.metadata, markdown = markdownText, html = it.html) } .map { noteRepository.create(userId, it) } - fun update(userId: Int, uuid: UUID, markdownText: String): Either<MarkdownParsingError, PersistedNote?> = - markdownConverter - .renderDocument(markdownText) + searcher.indexNote(userId, persistedNote) + persistedNote + } + + fun update(userId: Int, uuid: UUID, markdownText: String) = Either.fx<MarkdownParsingError, PersistedNote?> { + val persistedNote = !markdownConverter.renderDocument(markdownText) .map { it.copy(html = HtmlSanitizer.sanitize(it.html)) } .map { Note(it.metadata, markdown = markdownText, html = it.html) } .map { noteRepository.update(userId, uuid, it) } + persistedNote?.let { searcher.updateIndex(userId, it) } + persistedNote + } + fun paginatedNotes( userId: Int, page: Int, itemsPerPage: Int = 20, tag: String? = null, - deleted: Boolean = false + deleted: Boolean = false, ): PaginatedNotes { val count = noteRepository.count(userId, tag, deleted) val offset = (page - 1) * itemsPerPage @@ -44,11 +56,38 @@ class NoteService( } fun find(userId: Int, uuid: UUID) = noteRepository.find(userId, uuid) - fun trash(userId: Int, uuid: UUID) = noteRepository.delete(userId, uuid, permanent = false) - fun restore(userId: Int, uuid: UUID) = noteRepository.restore(userId, uuid) - fun delete(userId: Int, uuid: UUID) = noteRepository.delete(userId, uuid, permanent = true) + + fun trash(userId: Int, uuid: UUID): Boolean { + val res = noteRepository.delete(userId, uuid, permanent = false) + if (res) searcher.deleteIndex(userId, uuid) + return res + } + + fun restore(userId: Int, uuid: UUID): Boolean { + val res = noteRepository.restore(userId, uuid) + if (res) find(userId, uuid)?.let { note -> searcher.indexNote(userId, note) } + return res + } + + fun delete(userId: Int, uuid: UUID): Boolean { + val res = noteRepository.delete(userId, uuid, permanent = true) + if (res) searcher.deleteIndex(userId, uuid) + return res + } + fun countDeleted(userId: Int) = noteRepository.count(userId, deleted = true) + fun indexAll() { + val userIds = userRepository.findAll() + userIds.forEach { id -> + val notes = noteRepository.findAllDetails(id) + searcher.indexNotes(id, notes) + } + } + + fun search(userId: Int, searchTerms: SearchTerms) = searcher.search(userId, searchTerms) + + fun dropAllIndexes() = searcher.dropAll() } data class PaginatedNotes(val pages: Int, val notes: List<PersistedNoteMetadata>) diff --git a/domain/src/main/kotlin/usecases/repositories/NoteRepository.kt b/domain/src/main/kotlin/usecases/repositories/NoteRepository.kt index 7b5d58f..e961c9b 100644 --- a/domain/src/main/kotlin/usecases/repositories/NoteRepository.kt +++ b/domain/src/main/kotlin/usecases/repositories/NoteRepository.kt @@ -27,4 +27,5 @@ interface NoteRepository { fun find(userId: Int, uuid: UUID): PersistedNote? fun update(userId: Int, uuid: UUID, note: Note): PersistedNote? fun export(userId: Int): List<ExportedNote> + fun findAllDetails(userId: Int): List<PersistedNote> } diff --git a/domain/src/main/kotlin/usecases/repositories/UserRepository.kt b/domain/src/main/kotlin/usecases/repositories/UserRepository.kt index 2952e6f..1f52f34 100644 --- a/domain/src/main/kotlin/usecases/repositories/UserRepository.kt +++ b/domain/src/main/kotlin/usecases/repositories/UserRepository.kt @@ -10,4 +10,5 @@ interface UserRepository { fun exists(username: String): Boolean fun exists(id: Int): Boolean fun delete(id: Int): Boolean + fun findAll(): List<Int> } diff --git a/domain/src/main/kotlin/usecases/search/SearchUseCase.kt b/domain/src/main/kotlin/usecases/search/SearchUseCase.kt new file mode 100644 index 0000000..d2d406c --- /dev/null +++ b/domain/src/main/kotlin/usecases/search/SearchUseCase.kt @@ -0,0 +1,17 @@ +package be.simplenotes.domain.usecases.search + +import be.simplenotes.domain.model.PersistedNote +import be.simplenotes.domain.model.PersistedNoteMetadata +import java.util.* + +data class SearchTerms(val title: String?, val tag: String?, val content: String?) + +interface NoteSearcher { + fun indexNote(userId: Int, note: PersistedNote) + fun indexNotes(userId: Int, notes: List<PersistedNote>) + fun deleteIndex(userId: Int, uuid: UUID) + fun updateIndex(userId: Int, note: PersistedNote) + fun search(userId: Int, terms: SearchTerms): List<PersistedNoteMetadata> + fun dropIndex(userId: Int) + fun dropAll() +} diff --git a/persistance/src/main/kotlin/notes/NoteRepositoryImpl.kt b/persistance/src/main/kotlin/notes/NoteRepositoryImpl.kt index 7ab1a30..661cdf6 100644 --- a/persistance/src/main/kotlin/notes/NoteRepositoryImpl.kt +++ b/persistance/src/main/kotlin/notes/NoteRepositoryImpl.kt @@ -197,4 +197,25 @@ internal class NoteRepositoryImpl(private val db: Database) : NoteRepository { ) } } + + override fun findAllDetails(userId: Int): List<PersistedNote> { + val notes = db.notes + .filterColumns { it.columns - it.deleted } + .filter { (it.userId eq userId) and (it.deleted eq false) } + .toList() + + if (notes.isEmpty()) return emptyList() + + val uuids = notes.map { note -> note.uuid } + + val tagsByUuid = db.tags + .filterColumns { listOf(it.noteUuid, it.name) } + .filter { it.noteUuid inList uuids } + .groupByTo(HashMap(), { it.note.uuid }, { it.name }) + + return notes.map { note -> + val tags = tagsByUuid[note.uuid] ?: emptyList() + note.toPersistedNote(tags) + } + } } diff --git a/persistance/src/main/kotlin/users/UserRepositoryImpl.kt b/persistance/src/main/kotlin/users/UserRepositoryImpl.kt index b68397c..189eb59 100644 --- a/persistance/src/main/kotlin/users/UserRepositoryImpl.kt +++ b/persistance/src/main/kotlin/users/UserRepositoryImpl.kt @@ -28,4 +28,5 @@ internal class UserRepositoryImpl(private val db: Database) : UserRepository { override fun exists(username: String) = db.users.any { it.username eq username } override fun exists(id: Int) = db.users.any { it.id eq id } override fun delete(id: Int) = db.useTransaction { db.delete(Users) { it.id eq id } == 1 } + override fun findAll() = db.from(Users).select(Users.id).map { it[Users.id]!! } } diff --git a/persistance/src/main/resources/logback.xml b/persistance/src/main/resources/logback.xml index 5024926..d94bc53 100644 --- a/persistance/src/main/resources/logback.xml +++ b/persistance/src/main/resources/logback.xml @@ -6,11 +6,11 @@ </pattern> </encoder> </appender> - <root level="INFO"> + <root level="DEBUG"> <appender-ref ref="STDOUT"/> </root> <logger name="org.eclipse.jetty" level="INFO"/> - <logger name="me.liuwj.ktorm.database" level="DEBUG"/> + <logger name="me.liuwj.ktorm.database" level="INFO"/> <logger name="com.zaxxer.hikari" level="INFO"/> <logger name="org.flywaydb.core" level="INFO"/> </configuration> diff --git a/pom.xml b/pom.xml index 137781c..53b4e21 100644 --- a/pom.xml +++ b/pom.xml @@ -10,6 +10,7 @@ <module>app</module> <module>domain</module> <module>shared</module> + <module>search</module> </modules> <packaging>pom</packaging> diff --git a/search/pom.xml b/search/pom.xml new file mode 100644 index 0000000..4fb947f --- /dev/null +++ b/search/pom.xml @@ -0,0 +1,50 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>parent</artifactId> + <groupId>be.simplenotes</groupId> + <version>1.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>search</artifactId> + + <properties> + <lucene.version>8.5.2</lucene.version> + </properties> + + <dependencies> + <dependency> + <groupId>be.simplenotes</groupId> + <artifactId>domain</artifactId> + <version>1.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-core</artifactId> + <version>${lucene.version}</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-analyzers-common</artifactId> + <version>${lucene.version}</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-queryparser</artifactId> + <version>${lucene.version}</version> + </dependency> + + <dependency> + <groupId>be.simplenotes</groupId> + <artifactId>shared</artifactId> + <version>1.0-SNAPSHOT</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> + + +</project> diff --git a/search/src/main/kotlin/Constants.kt b/search/src/main/kotlin/Constants.kt new file mode 100644 index 0000000..c81f32a --- /dev/null +++ b/search/src/main/kotlin/Constants.kt @@ -0,0 +1,7 @@ +package be.simplenotes.search + +internal const val uuidField = "uuid" +internal const val titleField = "title" +internal const val tagsField = "tags" +internal const val contentField = "content" +internal const val updatedAtField = "updatedAt" diff --git a/search/src/main/kotlin/Extractors.kt b/search/src/main/kotlin/Extractors.kt new file mode 100644 index 0000000..ae1db61 --- /dev/null +++ b/search/src/main/kotlin/Extractors.kt @@ -0,0 +1,35 @@ +package be.simplenotes.search + +import be.simplenotes.domain.model.PersistedNote +import be.simplenotes.domain.model.PersistedNoteMetadata +import org.apache.lucene.document.Document +import org.apache.lucene.document.Field +import org.apache.lucene.document.StringField +import org.apache.lucene.document.TextField +import org.apache.lucene.search.IndexSearcher +import org.apache.lucene.search.TopDocs + +internal fun PersistedNote.toDocument(): Document { + val note = this + return Document().apply { + // non searchable fields + add(StringField(uuidField, UuidFieldConverter.toDoc(note.uuid), Field.Store.YES)) + add(StringField(updatedAtField, LocalDateTimeFieldConverter.toDoc(note.updatedAt), Field.Store.YES)) + + // searchable fields + add(TextField(titleField, note.meta.title, Field.Store.YES)) + add(TextField(tagsField, TagsFieldConverter.toDoc(note.meta.tags), Field.Store.YES)) + add(TextField(contentField, note.html, Field.Store.YES)) + } +} + +internal fun TopDocs.toResults(searcher: IndexSearcher) = scoreDocs.map { + searcher.doc(it.doc).let { doc -> + PersistedNoteMetadata( + title = doc.get(titleField), + uuid = UuidFieldConverter.fromDoc(doc.get(uuidField)), + updatedAt = LocalDateTimeFieldConverter.fromDoc(doc.get(updatedAtField)), + tags = TagsFieldConverter.fromDoc(doc.get(tagsField)) + ) + } +} diff --git a/search/src/main/kotlin/FieldConverters.kt b/search/src/main/kotlin/FieldConverters.kt new file mode 100644 index 0000000..34e3afa --- /dev/null +++ b/search/src/main/kotlin/FieldConverters.kt @@ -0,0 +1,26 @@ +package be.simplenotes.search + +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import java.util.* + +internal interface FieldConverter<T> { + fun toDoc(value: T): String + fun fromDoc(value: String): T +} + +internal object LocalDateTimeFieldConverter : FieldConverter<LocalDateTime> { + private val formatter = DateTimeFormatter.ISO_DATE_TIME + override fun toDoc(value: LocalDateTime): String = formatter.format(value) + override fun fromDoc(value: String): LocalDateTime = LocalDateTime.parse(value, formatter) +} + +internal object UuidFieldConverter : FieldConverter<UUID> { + override fun toDoc(value: UUID): String = value.toString() + override fun fromDoc(value: String): UUID = UUID.fromString(value) +} + +internal object TagsFieldConverter : FieldConverter<List<String>> { + override fun toDoc(value: List<String>): String = value.joinToString(" ") + override fun fromDoc(value: String): List<String> = value.split(" ") +} diff --git a/search/src/main/kotlin/NoteSearcherImpl.kt b/search/src/main/kotlin/NoteSearcherImpl.kt new file mode 100644 index 0000000..98f9237 --- /dev/null +++ b/search/src/main/kotlin/NoteSearcherImpl.kt @@ -0,0 +1,117 @@ +package be.simplenotes.search + +import be.simplenotes.domain.model.PersistedNote +import be.simplenotes.domain.model.PersistedNoteMetadata +import be.simplenotes.domain.usecases.search.NoteSearcher +import be.simplenotes.domain.usecases.search.SearchTerms +import be.simplenotes.search.utils.rmdir +import org.apache.lucene.analysis.standard.StandardAnalyzer +import org.apache.lucene.index.* +import org.apache.lucene.search.* +import org.apache.lucene.store.Directory +import org.apache.lucene.store.FSDirectory +import org.slf4j.LoggerFactory +import java.io.File +import java.nio.file.Path +import java.util.* + +class NoteSearcherImpl(basePath: Path = Path.of("/tmp", "lucene")) : NoteSearcher { + private val baseFile = basePath.toFile() + + private val logger = LoggerFactory.getLogger(javaClass) + + // region utils + private fun getDirectory(userId: Int): Directory { + val index = File(baseFile, userId.toString()).toPath() + return FSDirectory.open(index) + } + + private fun getIndexSearcher(userId: Int): IndexSearcher { + val directory = getDirectory(userId) + val reader: IndexReader = DirectoryReader.open(directory) + return IndexSearcher(reader) + } + // endregion + + override fun indexNote(userId: Int, note: PersistedNote) { + logger.debug("Indexing note ${note.uuid} for user $userId") + + val dir = getDirectory(userId) + val config = IndexWriterConfig(StandardAnalyzer()) + val writer = IndexWriter(dir, config) + val doc = note.toDocument() + + with(writer) { + addDocument(doc) + commit() + close() + } + } + + override fun indexNotes(userId: Int, notes: List<PersistedNote>) { + logger.debug("Indexing notes for user $userId") + + val dir = getDirectory(userId) + val config = IndexWriterConfig(StandardAnalyzer()) + val writer = IndexWriter(dir, config) + val docs = notes.map { it.toDocument() } + + with(writer) { + addDocuments(docs) + commit() + close() + } + } + + override fun deleteIndex(userId: Int, uuid: UUID) { + logger.debug("Deleting index $uuid for user $userId") + + val dir = getDirectory(userId) + val config = IndexWriterConfig(StandardAnalyzer()) + val writer = IndexWriter(dir, config) + + with(writer) { + deleteDocuments(TermQuery(Term(uuidField, UuidFieldConverter.toDoc(uuid)))) + commit() + close() + } + } + + override fun updateIndex(userId: Int, note: PersistedNote) { + logger.debug("Updating note ${note.uuid} for user $userId") + deleteIndex(userId, note.uuid) + indexNote(userId, note) + } + + override fun search(userId: Int, terms: SearchTerms): List<PersistedNoteMetadata> { + val searcher = getIndexSearcher(userId) + + val builder = BooleanQuery.Builder() + + terms.title?.let { + val titleQuery = FuzzyQuery(Term(titleField, it)) + builder.add(BooleanClause(titleQuery, BooleanClause.Occur.SHOULD)) + } + + terms.tag?.let { + val tagsQuery = FuzzyQuery(Term(tagsField, it)) + builder.add(BooleanClause(tagsQuery, BooleanClause.Occur.SHOULD)) + } + + terms.content?.let { + val contentQuery = FuzzyQuery(Term(contentField, it)) + builder.add(BooleanClause(contentQuery, BooleanClause.Occur.SHOULD)) + } + + val query = builder.build() + + val topDocs = searcher.search(query, 10) + logger.debug("Searching: `$query` results: ${topDocs.totalHits.value}") + return topDocs.toResults(searcher) + } + + override fun dropIndex(userId: Int) = rmdir(File(baseFile, userId.toString()).toPath()) + + override fun dropAll() = rmdir(baseFile.toPath()) + +} diff --git a/search/src/main/kotlin/SeachModule.kt b/search/src/main/kotlin/SeachModule.kt new file mode 100644 index 0000000..d86f399 --- /dev/null +++ b/search/src/main/kotlin/SeachModule.kt @@ -0,0 +1,9 @@ +package be.simplenotes.search + +import be.simplenotes.domain.usecases.search.NoteSearcher +import org.koin.dsl.module +import java.nio.file.Path + +val searchModule = module { + single<NoteSearcher> { NoteSearcherImpl(Path.of(".lucene")) } +} diff --git a/search/src/main/kotlin/utils/PathUtils.kt b/search/src/main/kotlin/utils/PathUtils.kt new file mode 100644 index 0000000..fd5966e --- /dev/null +++ b/search/src/main/kotlin/utils/PathUtils.kt @@ -0,0 +1,29 @@ +package be.simplenotes.search.utils + +import java.io.IOException +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes + +internal fun rmdir(path: Path) { + try { + Files.walkFileTree( + path, + object : SimpleFileVisitor<Path>() { + override fun visitFile(file: Path, attrs: BasicFileAttributes?): FileVisitResult { + Files.delete(file) + return FileVisitResult.CONTINUE + } + + override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult { + Files.delete(dir) + return FileVisitResult.CONTINUE + } + } + ) + } catch (e: IOException) { + // This is fine + } +} diff --git a/search/src/test/kotlin/NoteSearcherImplTest.kt b/search/src/test/kotlin/NoteSearcherImplTest.kt new file mode 100644 index 0000000..b6a5eae --- /dev/null +++ b/search/src/test/kotlin/NoteSearcherImplTest.kt @@ -0,0 +1,163 @@ +package be.simplenotes.search + +import be.simplenotes.domain.model.NoteMetadata +import be.simplenotes.domain.model.PersistedNote +import be.simplenotes.domain.model.PersistedNoteMetadata +import be.simplenotes.domain.usecases.search.SearchTerms +import org.assertj.core.api.Assertions.assertThat +import org.intellij.lang.annotations.Language +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.parallel.ResourceLock +import java.time.LocalDateTime +import java.util.* + +@ResourceLock("lucene") +internal class NoteSearcherImplTest { + + // region setup + private val searcher = NoteSearcherImpl() + + private fun index( + title: String, + tags: List<String> = emptyList(), + content: String = "", + uuid: UUID = UUID.randomUUID(), + ): PersistedNote { + val note = PersistedNote(NoteMetadata(title, tags), markdown = "", content, LocalDateTime.now(), uuid) + searcher.indexNote(1, note) + return note + } + + private fun search( + title: String? = null, + tag: String? = null, + content: String? = null, + ): List<PersistedNoteMetadata> = searcher.search(1, SearchTerms(title, tag, content)) + + @BeforeEach + @AfterAll + fun dropIndexes() { + searcher.dropIndex(1) + } + // endregion + + @Test + fun `exact title search`() { + index("first") + index("second") + index("flip") + + assertThat(search("first")) + .hasSizeGreaterThanOrEqualTo(1) + .anyMatch { it.title == "first" } + + assertThat(search("nothing")).isEmpty() + } + + @Test + fun `fuzzy title search`() { + index("first") + index("second") + index("flip") + + assertThat(search("firt")) + .hasSizeGreaterThanOrEqualTo(1) + .anyMatch { it.title == "first" } + + assertThat(search("nothing")).isEmpty() + } + + @Test + fun `exact tags search`() { + index("first", tags = listOf("example", "flamingo")) + index("second", tags = listOf("yes")) + index("second") + + assertThat(search(tag = "example")) + .hasSize(1) + .anyMatch { it.title == "first" } + } + + @Test + fun `exact content search`() { + @Language("html") + val content = + """ + <div> + <h1 class="title">Apache Lucene Core</h1> + <p>Apache Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> is a + high-performance, full-featured text search engine library written entirely in Java. + It is a technology suitable for nearly any application that requires full-text search, + especially cross-platform.</p> + <p>Apache Lucene is an open source project available for free download. Please use the + links on the right to access Lucene.</p> + <h1 id="lucenetm-features">Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> Features</h1> + <p>Lucene offers powerful features through a simple API:</p> + <h2 id="scalable-high-performance-indexing">Scalable, High-Performance Indexing</h2> + <ul> + <li>over <a href="http://home.apache.org/~mikemccand/lucenebench/indexing.html">150GB/hour on modern hardware</a></li> + <li>small RAM requirements -- only 1MB heap</li> + <li>incremental indexing as fast as batch indexing</li> + <li>index size roughly 20-30% the size of text indexed</li> + </ul> + """.trimIndent() + + index("first", content = content) + + assertThat(search(content = "fast")) + .hasSize(1) + .anyMatch { it.title == "first" } + + @Suppress("SpellCheckingInspection") + assertThat(search(content = "preformance")) // <- note the error + .hasSize(1) + .anyMatch { it.title == "first" } + } + + @Test + fun `combined search`() { + @Language("html") + val content = + """ + <div> + <h1 class="title">Apache Lucene Core</h1> + <p>Apache Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> is a + high-performance, full-featured text search engine library written entirely in Java. + It is a technology suitable for nearly any application that requires full-text search, + especially cross-platform.</p> + <p>Apache Lucene is an open source project available for free download. Please use the + links on the right to access Lucene.</p> + <h1 id="lucenetm-features">Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> Features</h1> + <p>Lucene offers powerful features through a simple API:</p> + <h2 id="scalable-high-performance-indexing">Scalable, High-Performance Indexing</h2> + <ul> + <li>over <a href="http://home.apache.org/~mikemccand/lucenebench/indexing.html">150GB/hour on modern hardware</a></li> + <li>small RAM requirements -- only 1MB heap</li> + <li>incremental indexing as fast as batch indexing</li> + <li>index size roughly 20-30% the size of text indexed</li> + </ul> + """.trimIndent() + + index("first", content = content, tags = listOf("abc")) + + assertThat(search(title = "fir", tag = "abc", content = "20")) + .hasSize(1) + } + + @Test + fun `delete index`() { + val uuid = index("first").uuid + searcher.deleteIndex(1, uuid) + assertThat(search("first")).isEmpty() + } + + @Test + fun `update index`() { + val note = index("first") + searcher.updateIndex(1, note.copy(meta = note.meta.copy(title = "new"))) + assertThat(search("first")).isEmpty() + assertThat(search("new")).hasSize(1) + } +}