Merge branch 'lucene-search'

This commit is contained in:
Hubert Van De Walle 2020-08-19 19:32:37 +02:00
commit 7305fb47c7
27 changed files with 684 additions and 29 deletions

3
.gitignore vendored
View File

@ -130,3 +130,6 @@ app/src/main/resources/static/styles*
# h2 db # h2 db
*.db *.db
# lucene index
.lucene/

View File

@ -8,6 +8,7 @@ COPY app/pom.xml app/pom.xml
COPY domain/pom.xml domain/pom.xml COPY domain/pom.xml domain/pom.xml
COPY persistance/pom.xml persistance/pom.xml COPY persistance/pom.xml persistance/pom.xml
COPY shared/pom.xml shared/pom.xml COPY shared/pom.xml shared/pom.xml
COPY search/pom.xml search/pom.xml
RUN mvn verify clean --fail-never RUN mvn verify clean --fail-never
@ -15,6 +16,7 @@ COPY app/src app/src
COPY domain/src domain/src COPY domain/src domain/src
COPY persistance/src persistance/src COPY persistance/src persistance/src
COPY shared/src shared/src COPY shared/src shared/src
COPY search/src search/src
RUN mvn -Dstyle.color=always package RUN mvn -Dstyle.color=always package

View File

@ -14,6 +14,11 @@
<artifactId>persistance</artifactId> <artifactId>persistance</artifactId>
<version>1.0-SNAPSHOT</version> <version>1.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency>
<groupId>be.simplenotes</groupId>
<artifactId>search</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency> <dependency>
<groupId>be.simplenotes</groupId> <groupId>be.simplenotes</groupId>
<artifactId>domain</artifactId> <artifactId>domain</artifactId>
@ -104,6 +109,12 @@
<include>**</include> <include>**</include>
</includes> </includes>
</filter> </filter>
<filter>
<artifact>org.apache.lucene:*</artifact>
<includes>
<include>**</include>
</includes>
</filter>
<filter> <filter>
<artifact>*:*</artifact> <artifact>*:*</artifact>
<excludes> <excludes>

View File

@ -14,8 +14,10 @@ import be.simplenotes.app.views.NoteView
import be.simplenotes.app.views.SettingView import be.simplenotes.app.views.SettingView
import be.simplenotes.app.views.UserView import be.simplenotes.app.views.UserView
import be.simplenotes.domain.domainModule import be.simplenotes.domain.domainModule
import be.simplenotes.domain.usecases.NoteService
import be.simplenotes.persistance.DbMigrations import be.simplenotes.persistance.DbMigrations
import be.simplenotes.persistance.persistanceModule import be.simplenotes.persistance.persistanceModule
import be.simplenotes.search.searchModule
import be.simplenotes.shared.config.DataSourceConfig import be.simplenotes.shared.config.DataSourceConfig
import be.simplenotes.shared.config.JwtConfig import be.simplenotes.shared.config.JwtConfig
import org.http4k.core.RequestContexts import org.http4k.core.RequestContexts
@ -36,6 +38,7 @@ fun main() {
baseModule, baseModule,
noteModule, noteModule,
settingsModule, settingsModule,
searchModule,
) )
}.koin }.koin
@ -50,6 +53,10 @@ fun main() {
val migrations = koin.get<DbMigrations>() val migrations = koin.get<DbMigrations>()
migrations.migrate() migrations.migrate()
val noteService = koin.get<NoteService>()
noteService.dropAllIndexes()
noteService.indexAll()
koin.get<Server>().start() koin.get<Server>().start()
} }

View File

@ -2,12 +2,14 @@ package be.simplenotes.app.controllers
import be.simplenotes.app.extensions.html import be.simplenotes.app.extensions.html
import be.simplenotes.app.extensions.redirect import be.simplenotes.app.extensions.redirect
import be.simplenotes.app.utils.parseSearchTerms
import be.simplenotes.app.views.NoteView import be.simplenotes.app.views.NoteView
import be.simplenotes.domain.security.JwtPayload import be.simplenotes.domain.security.JwtPayload
import be.simplenotes.domain.usecases.NoteService import be.simplenotes.domain.usecases.NoteService
import be.simplenotes.domain.usecases.markdown.InvalidMeta import be.simplenotes.domain.usecases.markdown.InvalidMeta
import be.simplenotes.domain.usecases.markdown.MissingMeta import be.simplenotes.domain.usecases.markdown.MissingMeta
import be.simplenotes.domain.usecases.markdown.ValidationError import be.simplenotes.domain.usecases.markdown.ValidationError
import be.simplenotes.domain.usecases.search.SearchTerms
import org.http4k.core.Method import org.http4k.core.Method
import org.http4k.core.Request import org.http4k.core.Request
import org.http4k.core.Response import org.http4k.core.Response
@ -34,7 +36,9 @@ class NoteController(
val html = when (it) { val html = when (it) {
MissingMeta -> view.noteEditor(jwtPayload, error = "Missing note metadata", textarea = markdownForm) MissingMeta -> view.noteEditor(jwtPayload, error = "Missing note metadata", textarea = markdownForm)
InvalidMeta -> view.noteEditor(jwtPayload, error = "Invalid note metadata", textarea = markdownForm) InvalidMeta -> view.noteEditor(jwtPayload, error = "Invalid note metadata", textarea = markdownForm)
is ValidationError -> view.noteEditor(jwtPayload, validationErrors = it.validationErrors, textarea = markdownForm) is ValidationError -> view.noteEditor(jwtPayload,
validationErrors = it.validationErrors,
textarea = markdownForm)
} }
Response(BAD_REQUEST).html(html) Response(BAD_REQUEST).html(html)
}, },
@ -52,6 +56,13 @@ class NoteController(
return Response(OK).html(view.notes(jwtPayload, notes, currentPage, pages, deletedCount, tag = tag)) return Response(OK).html(view.notes(jwtPayload, notes, currentPage, pages, deletedCount, tag = tag))
} }
fun search(request: Request, jwtPayload: JwtPayload): Response {
val terms = request.searchTerms()
val notes = noteService.search(jwtPayload.userId, terms)
val deletedCount = noteService.countDeleted(jwtPayload.userId)
return Response(OK).html(view.search(jwtPayload, notes, deletedCount))
}
fun note(request: Request, jwtPayload: JwtPayload): Response { fun note(request: Request, jwtPayload: JwtPayload): Response {
val noteUuid = request.uuidPath() ?: return Response(NOT_FOUND) val noteUuid = request.uuidPath() ?: return Response(NOT_FOUND)
@ -121,4 +132,6 @@ class NoteController(
null null
} }
} }
private fun Request.searchTerms(): SearchTerms = parseSearchTerms(form("search") ?: "")
} }

View File

@ -48,6 +48,7 @@ class Router(
"/settings" bind POST to { protected(it, settingsController::settings) }, "/settings" bind POST to { protected(it, settingsController::settings) },
"/export" bind POST to { protected(it, settingsController::export) }, "/export" bind POST to { protected(it, settingsController::export) },
"/notes" bind GET to { protected(it, noteController::list) }, "/notes" bind GET to { protected(it, noteController::list) },
"/notes" bind POST to { protected(it, noteController::search) },
"/notes/new" bind GET to { protected(it, noteController::new) }, "/notes/new" bind GET to { protected(it, noteController::new) },
"/notes/new" bind POST to { protected(it, noteController::new) }, "/notes/new" bind POST to { protected(it, noteController::new) },
"/notes/trash" bind GET to { protected(it, noteController::trash) }, "/notes/trash" bind GET to { protected(it, noteController::trash) },

View File

@ -0,0 +1,33 @@
package be.simplenotes.app.utils
import be.simplenotes.domain.usecases.search.SearchTerms
private val titleRe = """title:['"](?<title>.*?)['"]""".toRegex()
private val outerTitleRe = """(?<title>title:['"].*?['"])""".toRegex()
private val tagRe = """tag:['"](?<tag>.*?)['"]""".toRegex()
private val outerTagRe = """(?<tag>tag:['"].*?['"])""".toRegex()
fun parseSearchTerms(input: String): SearchTerms {
val title: String? = titleRe.find(input)?.groups?.get(1)?.value
val tag: String? = tagRe.find(input)?.groups?.get(1)?.value
var c: String = input
if (title != null) {
val titleGroup = outerTitleRe.find(input)?.groups?.get(1)?.value
titleGroup?.let { c = c.replace(it, "") }
}
if (tag != null) {
val tagGroup = outerTagRe.find(input)?.groups?.get(1)?.value
tagGroup?.let { c = c.replace(it, "") }
}
val content = c.trim().ifEmpty { null }
return SearchTerms(
title = title,
tag = tag,
content = content
)
}

View File

@ -55,20 +55,7 @@ class NoteView(staticFileResolver: StaticFileResolver) : View(staticFileResolver
tag: String?, tag: String?,
) = renderPage(title = "Notes", jwtPayload = jwtPayload) { ) = renderPage(title = "Notes", jwtPayload = jwtPayload) {
div("container mx-auto p-4") { div("container mx-auto p-4") {
div("flex justify-between mb-4") { noteListHeader(numberOfDeletedNotes)
h1("text-2xl underline") { +"Notes" }
span {
a(
href = "/notes/trash",
classes = "underline font-semibold"
) { +"Trash ($numberOfDeletedNotes)" }
a(
href = "/notes/new",
classes = "ml-2 btn btn-green"
) { +"New" }
}
}
if (notes.isNotEmpty()) if (notes.isNotEmpty())
noteTable(notes) noteTable(notes)
else else
@ -81,11 +68,22 @@ class NoteView(staticFileResolver: StaticFileResolver) : View(staticFileResolver
} }
} }
fun search(
jwtPayload: JwtPayload,
notes: List<PersistedNoteMetadata>,
numberOfDeletedNotes: Int,
) = renderPage("Notes", jwtPayload = jwtPayload) {
div("container mx-auto p-4") {
noteListHeader(numberOfDeletedNotes)
noteTable(notes)
}
}
fun trash( fun trash(
jwtPayload: JwtPayload, jwtPayload: JwtPayload,
notes: List<PersistedNoteMetadata>, notes: List<PersistedNoteMetadata>,
currentPage: Int, currentPage: Int,
numberOfPages: Int numberOfPages: Int,
) = renderPage(title = "Notes", jwtPayload = jwtPayload) { ) = renderPage(title = "Notes", jwtPayload = jwtPayload) {
div("container mx-auto p-4") { div("container mx-auto p-4") {
div("flex justify-between mb-4") { div("flex justify-between mb-4") {

View File

@ -0,0 +1,31 @@
package be.simplenotes.app.views.components
import kotlinx.html.*
import kotlinx.html.ButtonType.submit
import kotlinx.html.FormMethod.post
fun DIV.noteListHeader(numberOfDeletedNotes: Int) {
div("flex justify-between mb-4") {
h1("text-2xl underline") { +"Notes" }
span {
a(
href = "/notes/trash",
classes = "underline font-semibold"
) { +"Trash ($numberOfDeletedNotes)" }
a(
href = "/notes/new",
classes = "ml-2 btn btn-green"
) { +"New" }
}
}
form(method = post, classes = "mb-4") {
val colors = "bg-gray-800 border-gray-700 focus:border-teal-500 text-white"
input(
name = "search",
classes = "$colors rounded w-3/4 border appearance-none focus:outline-none text-base p-2"
)
button(type = submit, classes = "btn btn-green w-1/4") {
+"search"
}
}
}

View File

@ -0,0 +1,39 @@
package be.simplenotes.app.utils
import be.simplenotes.domain.usecases.search.SearchTerms
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import java.util.stream.Stream
internal class SearchTermsParserKtTest {
private fun createResult(
input: String,
title: String? = null,
tag: String? = null,
content: String? = null,
): Pair<String, SearchTerms> = input to SearchTerms(title, tag, content)
@Suppress("Unused")
private fun results() = Stream.of(
createResult("title:'example'", title = "example"),
createResult("title:'example with words'", title = "example with words"),
createResult("title:'example with words'", title = "example with words"),
createResult("""title:"double quotes"""", title = "double quotes"),
createResult("title:'example' something else", title = "example", content = "something else"),
createResult("tag:'example'", tag = "example"),
createResult("tag:'example' title:'other'", title = "other", tag = "example"),
createResult("blah blah tag:'example' title:'other'", title = "other", tag = "example", content = "blah blah"),
createResult("tag:'example' middle title:'other'", title = "other", tag = "example", content = "middle"),
createResult("tag:'example' title:'other' end", title = "other", tag = "example", content = "end"),
createResult("tag:'example abc' title:'other with words' this is the end ", title = "other with words", tag = "example abc", content = "this is the end"),
)
@ParameterizedTest
@MethodSource("results")
fun `valid search parser`(case: Pair<String, SearchTerms>) {
assertThat(parseSearchTerms(case.first)).isEqualTo(case.second)
}
}

View File

@ -26,7 +26,7 @@ val domainModule = module {
single<PasswordHash> { BcryptPasswordHash() } single<PasswordHash> { BcryptPasswordHash() }
single { SimpleJwt(get()) } single { SimpleJwt(get()) }
single { JwtPayloadExtractor(get()) } single { JwtPayloadExtractor(get()) }
single { NoteService(get(), get()) } single { NoteService(get(), get(), get(), get()) }
single<MarkdownConverter> { MarkdownConverterImpl() } single<MarkdownConverter> { MarkdownConverterImpl() }
single<ExportUseCase> { ExportUseCaseImpl(get()) } single<ExportUseCase> { ExportUseCaseImpl(get()) }
} }

View File

@ -1,6 +1,7 @@
package be.simplenotes.domain.usecases package be.simplenotes.domain.usecases
import arrow.core.Either import arrow.core.Either
import arrow.core.extensions.fx
import be.simplenotes.domain.model.Note import be.simplenotes.domain.model.Note
import be.simplenotes.domain.model.PersistedNote import be.simplenotes.domain.model.PersistedNote
import be.simplenotes.domain.model.PersistedNoteMetadata import be.simplenotes.domain.model.PersistedNoteMetadata
@ -8,33 +9,44 @@ import be.simplenotes.domain.security.HtmlSanitizer
import be.simplenotes.domain.usecases.markdown.MarkdownConverter import be.simplenotes.domain.usecases.markdown.MarkdownConverter
import be.simplenotes.domain.usecases.markdown.MarkdownParsingError import be.simplenotes.domain.usecases.markdown.MarkdownParsingError
import be.simplenotes.domain.usecases.repositories.NoteRepository import be.simplenotes.domain.usecases.repositories.NoteRepository
import be.simplenotes.domain.usecases.repositories.UserRepository
import be.simplenotes.domain.usecases.search.NoteSearcher
import be.simplenotes.domain.usecases.search.SearchTerms
import java.util.* import java.util.*
class NoteService( class NoteService(
private val markdownConverter: MarkdownConverter, private val markdownConverter: MarkdownConverter,
private val noteRepository: NoteRepository, private val noteRepository: NoteRepository,
private val userRepository: UserRepository,
private val searcher: NoteSearcher,
) { ) {
fun create(userId: Int, markdownText: String): Either<MarkdownParsingError, PersistedNote> = fun create(userId: Int, markdownText: String) = Either.fx<MarkdownParsingError, PersistedNote> {
markdownConverter val persistedNote = !markdownConverter.renderDocument(markdownText)
.renderDocument(markdownText)
.map { it.copy(html = HtmlSanitizer.sanitize(it.html)) } .map { it.copy(html = HtmlSanitizer.sanitize(it.html)) }
.map { Note(it.metadata, markdown = markdownText, html = it.html) } .map { Note(it.metadata, markdown = markdownText, html = it.html) }
.map { noteRepository.create(userId, it) } .map { noteRepository.create(userId, it) }
fun update(userId: Int, uuid: UUID, markdownText: String): Either<MarkdownParsingError, PersistedNote?> = searcher.indexNote(userId, persistedNote)
markdownConverter persistedNote
.renderDocument(markdownText) }
fun update(userId: Int, uuid: UUID, markdownText: String) = Either.fx<MarkdownParsingError, PersistedNote?> {
val persistedNote = !markdownConverter.renderDocument(markdownText)
.map { it.copy(html = HtmlSanitizer.sanitize(it.html)) } .map { it.copy(html = HtmlSanitizer.sanitize(it.html)) }
.map { Note(it.metadata, markdown = markdownText, html = it.html) } .map { Note(it.metadata, markdown = markdownText, html = it.html) }
.map { noteRepository.update(userId, uuid, it) } .map { noteRepository.update(userId, uuid, it) }
persistedNote?.let { searcher.updateIndex(userId, it) }
persistedNote
}
fun paginatedNotes( fun paginatedNotes(
userId: Int, userId: Int,
page: Int, page: Int,
itemsPerPage: Int = 20, itemsPerPage: Int = 20,
tag: String? = null, tag: String? = null,
deleted: Boolean = false deleted: Boolean = false,
): PaginatedNotes { ): PaginatedNotes {
val count = noteRepository.count(userId, tag, deleted) val count = noteRepository.count(userId, tag, deleted)
val offset = (page - 1) * itemsPerPage val offset = (page - 1) * itemsPerPage
@ -44,11 +56,38 @@ class NoteService(
} }
fun find(userId: Int, uuid: UUID) = noteRepository.find(userId, uuid) fun find(userId: Int, uuid: UUID) = noteRepository.find(userId, uuid)
fun trash(userId: Int, uuid: UUID) = noteRepository.delete(userId, uuid, permanent = false)
fun restore(userId: Int, uuid: UUID) = noteRepository.restore(userId, uuid) fun trash(userId: Int, uuid: UUID): Boolean {
fun delete(userId: Int, uuid: UUID) = noteRepository.delete(userId, uuid, permanent = true) val res = noteRepository.delete(userId, uuid, permanent = false)
if (res) searcher.deleteIndex(userId, uuid)
return res
}
fun restore(userId: Int, uuid: UUID): Boolean {
val res = noteRepository.restore(userId, uuid)
if (res) find(userId, uuid)?.let { note -> searcher.indexNote(userId, note) }
return res
}
fun delete(userId: Int, uuid: UUID): Boolean {
val res = noteRepository.delete(userId, uuid, permanent = true)
if (res) searcher.deleteIndex(userId, uuid)
return res
}
fun countDeleted(userId: Int) = noteRepository.count(userId, deleted = true) fun countDeleted(userId: Int) = noteRepository.count(userId, deleted = true)
fun indexAll() {
val userIds = userRepository.findAll()
userIds.forEach { id ->
val notes = noteRepository.findAllDetails(id)
searcher.indexNotes(id, notes)
}
}
fun search(userId: Int, searchTerms: SearchTerms) = searcher.search(userId, searchTerms)
fun dropAllIndexes() = searcher.dropAll()
} }
data class PaginatedNotes(val pages: Int, val notes: List<PersistedNoteMetadata>) data class PaginatedNotes(val pages: Int, val notes: List<PersistedNoteMetadata>)

View File

@ -27,4 +27,5 @@ interface NoteRepository {
fun find(userId: Int, uuid: UUID): PersistedNote? fun find(userId: Int, uuid: UUID): PersistedNote?
fun update(userId: Int, uuid: UUID, note: Note): PersistedNote? fun update(userId: Int, uuid: UUID, note: Note): PersistedNote?
fun export(userId: Int): List<ExportedNote> fun export(userId: Int): List<ExportedNote>
fun findAllDetails(userId: Int): List<PersistedNote>
} }

View File

@ -10,4 +10,5 @@ interface UserRepository {
fun exists(username: String): Boolean fun exists(username: String): Boolean
fun exists(id: Int): Boolean fun exists(id: Int): Boolean
fun delete(id: Int): Boolean fun delete(id: Int): Boolean
fun findAll(): List<Int>
} }

View File

@ -0,0 +1,17 @@
package be.simplenotes.domain.usecases.search
import be.simplenotes.domain.model.PersistedNote
import be.simplenotes.domain.model.PersistedNoteMetadata
import java.util.*
data class SearchTerms(val title: String?, val tag: String?, val content: String?)
interface NoteSearcher {
fun indexNote(userId: Int, note: PersistedNote)
fun indexNotes(userId: Int, notes: List<PersistedNote>)
fun deleteIndex(userId: Int, uuid: UUID)
fun updateIndex(userId: Int, note: PersistedNote)
fun search(userId: Int, terms: SearchTerms): List<PersistedNoteMetadata>
fun dropIndex(userId: Int)
fun dropAll()
}

View File

@ -197,4 +197,25 @@ internal class NoteRepositoryImpl(private val db: Database) : NoteRepository {
) )
} }
} }
override fun findAllDetails(userId: Int): List<PersistedNote> {
val notes = db.notes
.filterColumns { it.columns - it.deleted }
.filter { (it.userId eq userId) and (it.deleted eq false) }
.toList()
if (notes.isEmpty()) return emptyList()
val uuids = notes.map { note -> note.uuid }
val tagsByUuid = db.tags
.filterColumns { listOf(it.noteUuid, it.name) }
.filter { it.noteUuid inList uuids }
.groupByTo(HashMap(), { it.note.uuid }, { it.name })
return notes.map { note ->
val tags = tagsByUuid[note.uuid] ?: emptyList()
note.toPersistedNote(tags)
}
}
} }

View File

@ -28,4 +28,5 @@ internal class UserRepositoryImpl(private val db: Database) : UserRepository {
override fun exists(username: String) = db.users.any { it.username eq username } override fun exists(username: String) = db.users.any { it.username eq username }
override fun exists(id: Int) = db.users.any { it.id eq id } override fun exists(id: Int) = db.users.any { it.id eq id }
override fun delete(id: Int) = db.useTransaction { db.delete(Users) { it.id eq id } == 1 } override fun delete(id: Int) = db.useTransaction { db.delete(Users) { it.id eq id } == 1 }
override fun findAll() = db.from(Users).select(Users.id).map { it[Users.id]!! }
} }

View File

@ -6,11 +6,11 @@
</pattern> </pattern>
</encoder> </encoder>
</appender> </appender>
<root level="INFO"> <root level="DEBUG">
<appender-ref ref="STDOUT"/> <appender-ref ref="STDOUT"/>
</root> </root>
<logger name="org.eclipse.jetty" level="INFO"/> <logger name="org.eclipse.jetty" level="INFO"/>
<logger name="me.liuwj.ktorm.database" level="DEBUG"/> <logger name="me.liuwj.ktorm.database" level="INFO"/>
<logger name="com.zaxxer.hikari" level="INFO"/> <logger name="com.zaxxer.hikari" level="INFO"/>
<logger name="org.flywaydb.core" level="INFO"/> <logger name="org.flywaydb.core" level="INFO"/>
</configuration> </configuration>

View File

@ -10,6 +10,7 @@
<module>app</module> <module>app</module>
<module>domain</module> <module>domain</module>
<module>shared</module> <module>shared</module>
<module>search</module>
</modules> </modules>
<packaging>pom</packaging> <packaging>pom</packaging>

50
search/pom.xml Normal file
View File

@ -0,0 +1,50 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>parent</artifactId>
<groupId>be.simplenotes</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>search</artifactId>
<properties>
<lucene.version>8.5.2</lucene.version>
</properties>
<dependencies>
<dependency>
<groupId>be.simplenotes</groupId>
<artifactId>domain</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>be.simplenotes</groupId>
<artifactId>shared</artifactId>
<version>1.0-SNAPSHOT</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,7 @@
package be.simplenotes.search
internal const val uuidField = "uuid"
internal const val titleField = "title"
internal const val tagsField = "tags"
internal const val contentField = "content"
internal const val updatedAtField = "updatedAt"

View File

@ -0,0 +1,35 @@
package be.simplenotes.search
import be.simplenotes.domain.model.PersistedNote
import be.simplenotes.domain.model.PersistedNoteMetadata
import org.apache.lucene.document.Document
import org.apache.lucene.document.Field
import org.apache.lucene.document.StringField
import org.apache.lucene.document.TextField
import org.apache.lucene.search.IndexSearcher
import org.apache.lucene.search.TopDocs
internal fun PersistedNote.toDocument(): Document {
val note = this
return Document().apply {
// non searchable fields
add(StringField(uuidField, UuidFieldConverter.toDoc(note.uuid), Field.Store.YES))
add(StringField(updatedAtField, LocalDateTimeFieldConverter.toDoc(note.updatedAt), Field.Store.YES))
// searchable fields
add(TextField(titleField, note.meta.title, Field.Store.YES))
add(TextField(tagsField, TagsFieldConverter.toDoc(note.meta.tags), Field.Store.YES))
add(TextField(contentField, note.html, Field.Store.YES))
}
}
internal fun TopDocs.toResults(searcher: IndexSearcher) = scoreDocs.map {
searcher.doc(it.doc).let { doc ->
PersistedNoteMetadata(
title = doc.get(titleField),
uuid = UuidFieldConverter.fromDoc(doc.get(uuidField)),
updatedAt = LocalDateTimeFieldConverter.fromDoc(doc.get(updatedAtField)),
tags = TagsFieldConverter.fromDoc(doc.get(tagsField))
)
}
}

View File

@ -0,0 +1,26 @@
package be.simplenotes.search
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import java.util.*
internal interface FieldConverter<T> {
fun toDoc(value: T): String
fun fromDoc(value: String): T
}
internal object LocalDateTimeFieldConverter : FieldConverter<LocalDateTime> {
private val formatter = DateTimeFormatter.ISO_DATE_TIME
override fun toDoc(value: LocalDateTime): String = formatter.format(value)
override fun fromDoc(value: String): LocalDateTime = LocalDateTime.parse(value, formatter)
}
internal object UuidFieldConverter : FieldConverter<UUID> {
override fun toDoc(value: UUID): String = value.toString()
override fun fromDoc(value: String): UUID = UUID.fromString(value)
}
internal object TagsFieldConverter : FieldConverter<List<String>> {
override fun toDoc(value: List<String>): String = value.joinToString(" ")
override fun fromDoc(value: String): List<String> = value.split(" ")
}

View File

@ -0,0 +1,117 @@
package be.simplenotes.search
import be.simplenotes.domain.model.PersistedNote
import be.simplenotes.domain.model.PersistedNoteMetadata
import be.simplenotes.domain.usecases.search.NoteSearcher
import be.simplenotes.domain.usecases.search.SearchTerms
import be.simplenotes.search.utils.rmdir
import org.apache.lucene.analysis.standard.StandardAnalyzer
import org.apache.lucene.index.*
import org.apache.lucene.search.*
import org.apache.lucene.store.Directory
import org.apache.lucene.store.FSDirectory
import org.slf4j.LoggerFactory
import java.io.File
import java.nio.file.Path
import java.util.*
class NoteSearcherImpl(basePath: Path = Path.of("/tmp", "lucene")) : NoteSearcher {
private val baseFile = basePath.toFile()
private val logger = LoggerFactory.getLogger(javaClass)
// region utils
private fun getDirectory(userId: Int): Directory {
val index = File(baseFile, userId.toString()).toPath()
return FSDirectory.open(index)
}
private fun getIndexSearcher(userId: Int): IndexSearcher {
val directory = getDirectory(userId)
val reader: IndexReader = DirectoryReader.open(directory)
return IndexSearcher(reader)
}
// endregion
override fun indexNote(userId: Int, note: PersistedNote) {
logger.debug("Indexing note ${note.uuid} for user $userId")
val dir = getDirectory(userId)
val config = IndexWriterConfig(StandardAnalyzer())
val writer = IndexWriter(dir, config)
val doc = note.toDocument()
with(writer) {
addDocument(doc)
commit()
close()
}
}
override fun indexNotes(userId: Int, notes: List<PersistedNote>) {
logger.debug("Indexing notes for user $userId")
val dir = getDirectory(userId)
val config = IndexWriterConfig(StandardAnalyzer())
val writer = IndexWriter(dir, config)
val docs = notes.map { it.toDocument() }
with(writer) {
addDocuments(docs)
commit()
close()
}
}
override fun deleteIndex(userId: Int, uuid: UUID) {
logger.debug("Deleting index $uuid for user $userId")
val dir = getDirectory(userId)
val config = IndexWriterConfig(StandardAnalyzer())
val writer = IndexWriter(dir, config)
with(writer) {
deleteDocuments(TermQuery(Term(uuidField, UuidFieldConverter.toDoc(uuid))))
commit()
close()
}
}
override fun updateIndex(userId: Int, note: PersistedNote) {
logger.debug("Updating note ${note.uuid} for user $userId")
deleteIndex(userId, note.uuid)
indexNote(userId, note)
}
override fun search(userId: Int, terms: SearchTerms): List<PersistedNoteMetadata> {
val searcher = getIndexSearcher(userId)
val builder = BooleanQuery.Builder()
terms.title?.let {
val titleQuery = FuzzyQuery(Term(titleField, it))
builder.add(BooleanClause(titleQuery, BooleanClause.Occur.SHOULD))
}
terms.tag?.let {
val tagsQuery = FuzzyQuery(Term(tagsField, it))
builder.add(BooleanClause(tagsQuery, BooleanClause.Occur.SHOULD))
}
terms.content?.let {
val contentQuery = FuzzyQuery(Term(contentField, it))
builder.add(BooleanClause(contentQuery, BooleanClause.Occur.SHOULD))
}
val query = builder.build()
val topDocs = searcher.search(query, 10)
logger.debug("Searching: `$query` results: ${topDocs.totalHits.value}")
return topDocs.toResults(searcher)
}
override fun dropIndex(userId: Int) = rmdir(File(baseFile, userId.toString()).toPath())
override fun dropAll() = rmdir(baseFile.toPath())
}

View File

@ -0,0 +1,9 @@
package be.simplenotes.search
import be.simplenotes.domain.usecases.search.NoteSearcher
import org.koin.dsl.module
import java.nio.file.Path
val searchModule = module {
single<NoteSearcher> { NoteSearcherImpl(Path.of(".lucene")) }
}

View File

@ -0,0 +1,29 @@
package be.simplenotes.search.utils
import java.io.IOException
import java.nio.file.FileVisitResult
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
internal fun rmdir(path: Path) {
try {
Files.walkFileTree(
path,
object : SimpleFileVisitor<Path>() {
override fun visitFile(file: Path, attrs: BasicFileAttributes?): FileVisitResult {
Files.delete(file)
return FileVisitResult.CONTINUE
}
override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult {
Files.delete(dir)
return FileVisitResult.CONTINUE
}
}
)
} catch (e: IOException) {
// This is fine
}
}

View File

@ -0,0 +1,163 @@
package be.simplenotes.search
import be.simplenotes.domain.model.NoteMetadata
import be.simplenotes.domain.model.PersistedNote
import be.simplenotes.domain.model.PersistedNoteMetadata
import be.simplenotes.domain.usecases.search.SearchTerms
import org.assertj.core.api.Assertions.assertThat
import org.intellij.lang.annotations.Language
import org.junit.jupiter.api.AfterAll
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.parallel.ResourceLock
import java.time.LocalDateTime
import java.util.*
@ResourceLock("lucene")
internal class NoteSearcherImplTest {
// region setup
private val searcher = NoteSearcherImpl()
private fun index(
title: String,
tags: List<String> = emptyList(),
content: String = "",
uuid: UUID = UUID.randomUUID(),
): PersistedNote {
val note = PersistedNote(NoteMetadata(title, tags), markdown = "", content, LocalDateTime.now(), uuid)
searcher.indexNote(1, note)
return note
}
private fun search(
title: String? = null,
tag: String? = null,
content: String? = null,
): List<PersistedNoteMetadata> = searcher.search(1, SearchTerms(title, tag, content))
@BeforeEach
@AfterAll
fun dropIndexes() {
searcher.dropIndex(1)
}
// endregion
@Test
fun `exact title search`() {
index("first")
index("second")
index("flip")
assertThat(search("first"))
.hasSizeGreaterThanOrEqualTo(1)
.anyMatch { it.title == "first" }
assertThat(search("nothing")).isEmpty()
}
@Test
fun `fuzzy title search`() {
index("first")
index("second")
index("flip")
assertThat(search("firt"))
.hasSizeGreaterThanOrEqualTo(1)
.anyMatch { it.title == "first" }
assertThat(search("nothing")).isEmpty()
}
@Test
fun `exact tags search`() {
index("first", tags = listOf("example", "flamingo"))
index("second", tags = listOf("yes"))
index("second")
assertThat(search(tag = "example"))
.hasSize(1)
.anyMatch { it.title == "first" }
}
@Test
fun `exact content search`() {
@Language("html")
val content =
"""
<div>
<h1 class="title">Apache Lucene Core</h1>
<p>Apache Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> is a
high-performance, full-featured text search engine library written entirely in Java.
It is a technology suitable for nearly any application that requires full-text search,
especially cross-platform.</p>
<p>Apache Lucene is an open source project available for free download. Please use the
links on the right to access Lucene.</p>
<h1 id="lucenetm-features">Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> Features</h1>
<p>Lucene offers powerful features through a simple API:</p>
<h2 id="scalable-high-performance-indexing">Scalable, High-Performance Indexing</h2>
<ul>
<li>over <a href="http://home.apache.org/~mikemccand/lucenebench/indexing.html">150GB/hour on modern hardware</a></li>
<li>small RAM requirements -- only 1MB heap</li>
<li>incremental indexing as fast as batch indexing</li>
<li>index size roughly 20-30% the size of text indexed</li>
</ul>
""".trimIndent()
index("first", content = content)
assertThat(search(content = "fast"))
.hasSize(1)
.anyMatch { it.title == "first" }
@Suppress("SpellCheckingInspection")
assertThat(search(content = "preformance")) // <- note the error
.hasSize(1)
.anyMatch { it.title == "first" }
}
@Test
fun `combined search`() {
@Language("html")
val content =
"""
<div>
<h1 class="title">Apache Lucene Core</h1>
<p>Apache Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> is a
high-performance, full-featured text search engine library written entirely in Java.
It is a technology suitable for nearly any application that requires full-text search,
especially cross-platform.</p>
<p>Apache Lucene is an open source project available for free download. Please use the
links on the right to access Lucene.</p>
<h1 id="lucenetm-features">Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> Features</h1>
<p>Lucene offers powerful features through a simple API:</p>
<h2 id="scalable-high-performance-indexing">Scalable, High-Performance Indexing</h2>
<ul>
<li>over <a href="http://home.apache.org/~mikemccand/lucenebench/indexing.html">150GB/hour on modern hardware</a></li>
<li>small RAM requirements -- only 1MB heap</li>
<li>incremental indexing as fast as batch indexing</li>
<li>index size roughly 20-30% the size of text indexed</li>
</ul>
""".trimIndent()
index("first", content = content, tags = listOf("abc"))
assertThat(search(title = "fir", tag = "abc", content = "20"))
.hasSize(1)
}
@Test
fun `delete index`() {
val uuid = index("first").uuid
searcher.deleteIndex(1, uuid)
assertThat(search("first")).isEmpty()
}
@Test
fun `update index`() {
val note = index("first")
searcher.updateIndex(1, note.copy(meta = note.meta.copy(title = "new")))
assertThat(search("first")).isEmpty()
assertThat(search("new")).hasSize(1)
}
}