Merge branch 'lucene-search'
This commit is contained in:
commit
7305fb47c7
3
.gitignore
vendored
3
.gitignore
vendored
@ -130,3 +130,6 @@ app/src/main/resources/static/styles*
|
||||
|
||||
# h2 db
|
||||
*.db
|
||||
|
||||
# lucene index
|
||||
.lucene/
|
||||
|
||||
@ -8,6 +8,7 @@ COPY app/pom.xml app/pom.xml
|
||||
COPY domain/pom.xml domain/pom.xml
|
||||
COPY persistance/pom.xml persistance/pom.xml
|
||||
COPY shared/pom.xml shared/pom.xml
|
||||
COPY search/pom.xml search/pom.xml
|
||||
|
||||
RUN mvn verify clean --fail-never
|
||||
|
||||
@ -15,6 +16,7 @@ COPY app/src app/src
|
||||
COPY domain/src domain/src
|
||||
COPY persistance/src persistance/src
|
||||
COPY shared/src shared/src
|
||||
COPY search/src search/src
|
||||
|
||||
RUN mvn -Dstyle.color=always package
|
||||
|
||||
|
||||
11
app/pom.xml
11
app/pom.xml
@ -14,6 +14,11 @@
|
||||
<artifactId>persistance</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>be.simplenotes</groupId>
|
||||
<artifactId>search</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>be.simplenotes</groupId>
|
||||
<artifactId>domain</artifactId>
|
||||
@ -104,6 +109,12 @@
|
||||
<include>**</include>
|
||||
</includes>
|
||||
</filter>
|
||||
<filter>
|
||||
<artifact>org.apache.lucene:*</artifact>
|
||||
<includes>
|
||||
<include>**</include>
|
||||
</includes>
|
||||
</filter>
|
||||
<filter>
|
||||
<artifact>*:*</artifact>
|
||||
<excludes>
|
||||
|
||||
@ -14,8 +14,10 @@ import be.simplenotes.app.views.NoteView
|
||||
import be.simplenotes.app.views.SettingView
|
||||
import be.simplenotes.app.views.UserView
|
||||
import be.simplenotes.domain.domainModule
|
||||
import be.simplenotes.domain.usecases.NoteService
|
||||
import be.simplenotes.persistance.DbMigrations
|
||||
import be.simplenotes.persistance.persistanceModule
|
||||
import be.simplenotes.search.searchModule
|
||||
import be.simplenotes.shared.config.DataSourceConfig
|
||||
import be.simplenotes.shared.config.JwtConfig
|
||||
import org.http4k.core.RequestContexts
|
||||
@ -36,6 +38,7 @@ fun main() {
|
||||
baseModule,
|
||||
noteModule,
|
||||
settingsModule,
|
||||
searchModule,
|
||||
)
|
||||
}.koin
|
||||
|
||||
@ -50,6 +53,10 @@ fun main() {
|
||||
val migrations = koin.get<DbMigrations>()
|
||||
migrations.migrate()
|
||||
|
||||
val noteService = koin.get<NoteService>()
|
||||
noteService.dropAllIndexes()
|
||||
noteService.indexAll()
|
||||
|
||||
koin.get<Server>().start()
|
||||
}
|
||||
|
||||
|
||||
@ -2,12 +2,14 @@ package be.simplenotes.app.controllers
|
||||
|
||||
import be.simplenotes.app.extensions.html
|
||||
import be.simplenotes.app.extensions.redirect
|
||||
import be.simplenotes.app.utils.parseSearchTerms
|
||||
import be.simplenotes.app.views.NoteView
|
||||
import be.simplenotes.domain.security.JwtPayload
|
||||
import be.simplenotes.domain.usecases.NoteService
|
||||
import be.simplenotes.domain.usecases.markdown.InvalidMeta
|
||||
import be.simplenotes.domain.usecases.markdown.MissingMeta
|
||||
import be.simplenotes.domain.usecases.markdown.ValidationError
|
||||
import be.simplenotes.domain.usecases.search.SearchTerms
|
||||
import org.http4k.core.Method
|
||||
import org.http4k.core.Request
|
||||
import org.http4k.core.Response
|
||||
@ -34,7 +36,9 @@ class NoteController(
|
||||
val html = when (it) {
|
||||
MissingMeta -> view.noteEditor(jwtPayload, error = "Missing note metadata", textarea = markdownForm)
|
||||
InvalidMeta -> view.noteEditor(jwtPayload, error = "Invalid note metadata", textarea = markdownForm)
|
||||
is ValidationError -> view.noteEditor(jwtPayload, validationErrors = it.validationErrors, textarea = markdownForm)
|
||||
is ValidationError -> view.noteEditor(jwtPayload,
|
||||
validationErrors = it.validationErrors,
|
||||
textarea = markdownForm)
|
||||
}
|
||||
Response(BAD_REQUEST).html(html)
|
||||
},
|
||||
@ -52,6 +56,13 @@ class NoteController(
|
||||
return Response(OK).html(view.notes(jwtPayload, notes, currentPage, pages, deletedCount, tag = tag))
|
||||
}
|
||||
|
||||
fun search(request: Request, jwtPayload: JwtPayload): Response {
|
||||
val terms = request.searchTerms()
|
||||
val notes = noteService.search(jwtPayload.userId, terms)
|
||||
val deletedCount = noteService.countDeleted(jwtPayload.userId)
|
||||
return Response(OK).html(view.search(jwtPayload, notes, deletedCount))
|
||||
}
|
||||
|
||||
fun note(request: Request, jwtPayload: JwtPayload): Response {
|
||||
val noteUuid = request.uuidPath() ?: return Response(NOT_FOUND)
|
||||
|
||||
@ -121,4 +132,6 @@ class NoteController(
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun Request.searchTerms(): SearchTerms = parseSearchTerms(form("search") ?: "")
|
||||
}
|
||||
|
||||
@ -48,6 +48,7 @@ class Router(
|
||||
"/settings" bind POST to { protected(it, settingsController::settings) },
|
||||
"/export" bind POST to { protected(it, settingsController::export) },
|
||||
"/notes" bind GET to { protected(it, noteController::list) },
|
||||
"/notes" bind POST to { protected(it, noteController::search) },
|
||||
"/notes/new" bind GET to { protected(it, noteController::new) },
|
||||
"/notes/new" bind POST to { protected(it, noteController::new) },
|
||||
"/notes/trash" bind GET to { protected(it, noteController::trash) },
|
||||
|
||||
33
app/src/main/kotlin/utils/SearchTermsParser.kt
Normal file
33
app/src/main/kotlin/utils/SearchTermsParser.kt
Normal file
@ -0,0 +1,33 @@
|
||||
package be.simplenotes.app.utils
|
||||
|
||||
import be.simplenotes.domain.usecases.search.SearchTerms
|
||||
|
||||
private val titleRe = """title:['"](?<title>.*?)['"]""".toRegex()
|
||||
private val outerTitleRe = """(?<title>title:['"].*?['"])""".toRegex()
|
||||
|
||||
private val tagRe = """tag:['"](?<tag>.*?)['"]""".toRegex()
|
||||
private val outerTagRe = """(?<tag>tag:['"].*?['"])""".toRegex()
|
||||
|
||||
fun parseSearchTerms(input: String): SearchTerms {
|
||||
val title: String? = titleRe.find(input)?.groups?.get(1)?.value
|
||||
val tag: String? = tagRe.find(input)?.groups?.get(1)?.value
|
||||
var c: String = input
|
||||
|
||||
if (title != null) {
|
||||
val titleGroup = outerTitleRe.find(input)?.groups?.get(1)?.value
|
||||
titleGroup?.let { c = c.replace(it, "") }
|
||||
}
|
||||
|
||||
if (tag != null) {
|
||||
val tagGroup = outerTagRe.find(input)?.groups?.get(1)?.value
|
||||
tagGroup?.let { c = c.replace(it, "") }
|
||||
}
|
||||
|
||||
val content = c.trim().ifEmpty { null }
|
||||
|
||||
return SearchTerms(
|
||||
title = title,
|
||||
tag = tag,
|
||||
content = content
|
||||
)
|
||||
}
|
||||
@ -55,20 +55,7 @@ class NoteView(staticFileResolver: StaticFileResolver) : View(staticFileResolver
|
||||
tag: String?,
|
||||
) = renderPage(title = "Notes", jwtPayload = jwtPayload) {
|
||||
div("container mx-auto p-4") {
|
||||
div("flex justify-between mb-4") {
|
||||
h1("text-2xl underline") { +"Notes" }
|
||||
span {
|
||||
a(
|
||||
href = "/notes/trash",
|
||||
classes = "underline font-semibold"
|
||||
) { +"Trash ($numberOfDeletedNotes)" }
|
||||
a(
|
||||
href = "/notes/new",
|
||||
classes = "ml-2 btn btn-green"
|
||||
) { +"New" }
|
||||
}
|
||||
|
||||
}
|
||||
noteListHeader(numberOfDeletedNotes)
|
||||
if (notes.isNotEmpty())
|
||||
noteTable(notes)
|
||||
else
|
||||
@ -81,11 +68,22 @@ class NoteView(staticFileResolver: StaticFileResolver) : View(staticFileResolver
|
||||
}
|
||||
}
|
||||
|
||||
fun search(
|
||||
jwtPayload: JwtPayload,
|
||||
notes: List<PersistedNoteMetadata>,
|
||||
numberOfDeletedNotes: Int,
|
||||
) = renderPage("Notes", jwtPayload = jwtPayload) {
|
||||
div("container mx-auto p-4") {
|
||||
noteListHeader(numberOfDeletedNotes)
|
||||
noteTable(notes)
|
||||
}
|
||||
}
|
||||
|
||||
fun trash(
|
||||
jwtPayload: JwtPayload,
|
||||
notes: List<PersistedNoteMetadata>,
|
||||
currentPage: Int,
|
||||
numberOfPages: Int
|
||||
numberOfPages: Int,
|
||||
) = renderPage(title = "Notes", jwtPayload = jwtPayload) {
|
||||
div("container mx-auto p-4") {
|
||||
div("flex justify-between mb-4") {
|
||||
|
||||
31
app/src/main/kotlin/views/components/NoteListHeader.kt
Normal file
31
app/src/main/kotlin/views/components/NoteListHeader.kt
Normal file
@ -0,0 +1,31 @@
|
||||
package be.simplenotes.app.views.components
|
||||
|
||||
import kotlinx.html.*
|
||||
import kotlinx.html.ButtonType.submit
|
||||
import kotlinx.html.FormMethod.post
|
||||
|
||||
fun DIV.noteListHeader(numberOfDeletedNotes: Int) {
|
||||
div("flex justify-between mb-4") {
|
||||
h1("text-2xl underline") { +"Notes" }
|
||||
span {
|
||||
a(
|
||||
href = "/notes/trash",
|
||||
classes = "underline font-semibold"
|
||||
) { +"Trash ($numberOfDeletedNotes)" }
|
||||
a(
|
||||
href = "/notes/new",
|
||||
classes = "ml-2 btn btn-green"
|
||||
) { +"New" }
|
||||
}
|
||||
}
|
||||
form(method = post, classes = "mb-4") {
|
||||
val colors = "bg-gray-800 border-gray-700 focus:border-teal-500 text-white"
|
||||
input(
|
||||
name = "search",
|
||||
classes = "$colors rounded w-3/4 border appearance-none focus:outline-none text-base p-2"
|
||||
)
|
||||
button(type = submit, classes = "btn btn-green w-1/4") {
|
||||
+"search"
|
||||
}
|
||||
}
|
||||
}
|
||||
39
app/src/test/kotlin/utils/SearchTermsParserKtTest.kt
Normal file
39
app/src/test/kotlin/utils/SearchTermsParserKtTest.kt
Normal file
@ -0,0 +1,39 @@
|
||||
package be.simplenotes.app.utils
|
||||
|
||||
import be.simplenotes.domain.usecases.search.SearchTerms
|
||||
import org.assertj.core.api.Assertions.assertThat
|
||||
import org.junit.jupiter.params.ParameterizedTest
|
||||
import org.junit.jupiter.params.provider.MethodSource
|
||||
import java.util.stream.Stream
|
||||
|
||||
internal class SearchTermsParserKtTest {
|
||||
|
||||
private fun createResult(
|
||||
input: String,
|
||||
title: String? = null,
|
||||
tag: String? = null,
|
||||
content: String? = null,
|
||||
): Pair<String, SearchTerms> = input to SearchTerms(title, tag, content)
|
||||
|
||||
@Suppress("Unused")
|
||||
private fun results() = Stream.of(
|
||||
createResult("title:'example'", title = "example"),
|
||||
createResult("title:'example with words'", title = "example with words"),
|
||||
createResult("title:'example with words'", title = "example with words"),
|
||||
createResult("""title:"double quotes"""", title = "double quotes"),
|
||||
createResult("title:'example' something else", title = "example", content = "something else"),
|
||||
createResult("tag:'example'", tag = "example"),
|
||||
createResult("tag:'example' title:'other'", title = "other", tag = "example"),
|
||||
createResult("blah blah tag:'example' title:'other'", title = "other", tag = "example", content = "blah blah"),
|
||||
createResult("tag:'example' middle title:'other'", title = "other", tag = "example", content = "middle"),
|
||||
createResult("tag:'example' title:'other' end", title = "other", tag = "example", content = "end"),
|
||||
createResult("tag:'example abc' title:'other with words' this is the end ", title = "other with words", tag = "example abc", content = "this is the end"),
|
||||
)
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("results")
|
||||
fun `valid search parser`(case: Pair<String, SearchTerms>) {
|
||||
assertThat(parseSearchTerms(case.first)).isEqualTo(case.second)
|
||||
}
|
||||
|
||||
}
|
||||
@ -26,7 +26,7 @@ val domainModule = module {
|
||||
single<PasswordHash> { BcryptPasswordHash() }
|
||||
single { SimpleJwt(get()) }
|
||||
single { JwtPayloadExtractor(get()) }
|
||||
single { NoteService(get(), get()) }
|
||||
single { NoteService(get(), get(), get(), get()) }
|
||||
single<MarkdownConverter> { MarkdownConverterImpl() }
|
||||
single<ExportUseCase> { ExportUseCaseImpl(get()) }
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package be.simplenotes.domain.usecases
|
||||
|
||||
import arrow.core.Either
|
||||
import arrow.core.extensions.fx
|
||||
import be.simplenotes.domain.model.Note
|
||||
import be.simplenotes.domain.model.PersistedNote
|
||||
import be.simplenotes.domain.model.PersistedNoteMetadata
|
||||
@ -8,33 +9,44 @@ import be.simplenotes.domain.security.HtmlSanitizer
|
||||
import be.simplenotes.domain.usecases.markdown.MarkdownConverter
|
||||
import be.simplenotes.domain.usecases.markdown.MarkdownParsingError
|
||||
import be.simplenotes.domain.usecases.repositories.NoteRepository
|
||||
import be.simplenotes.domain.usecases.repositories.UserRepository
|
||||
import be.simplenotes.domain.usecases.search.NoteSearcher
|
||||
import be.simplenotes.domain.usecases.search.SearchTerms
|
||||
import java.util.*
|
||||
|
||||
class NoteService(
|
||||
private val markdownConverter: MarkdownConverter,
|
||||
private val noteRepository: NoteRepository,
|
||||
private val userRepository: UserRepository,
|
||||
private val searcher: NoteSearcher,
|
||||
) {
|
||||
|
||||
fun create(userId: Int, markdownText: String): Either<MarkdownParsingError, PersistedNote> =
|
||||
markdownConverter
|
||||
.renderDocument(markdownText)
|
||||
fun create(userId: Int, markdownText: String) = Either.fx<MarkdownParsingError, PersistedNote> {
|
||||
val persistedNote = !markdownConverter.renderDocument(markdownText)
|
||||
.map { it.copy(html = HtmlSanitizer.sanitize(it.html)) }
|
||||
.map { Note(it.metadata, markdown = markdownText, html = it.html) }
|
||||
.map { noteRepository.create(userId, it) }
|
||||
|
||||
fun update(userId: Int, uuid: UUID, markdownText: String): Either<MarkdownParsingError, PersistedNote?> =
|
||||
markdownConverter
|
||||
.renderDocument(markdownText)
|
||||
searcher.indexNote(userId, persistedNote)
|
||||
persistedNote
|
||||
}
|
||||
|
||||
fun update(userId: Int, uuid: UUID, markdownText: String) = Either.fx<MarkdownParsingError, PersistedNote?> {
|
||||
val persistedNote = !markdownConverter.renderDocument(markdownText)
|
||||
.map { it.copy(html = HtmlSanitizer.sanitize(it.html)) }
|
||||
.map { Note(it.metadata, markdown = markdownText, html = it.html) }
|
||||
.map { noteRepository.update(userId, uuid, it) }
|
||||
|
||||
persistedNote?.let { searcher.updateIndex(userId, it) }
|
||||
persistedNote
|
||||
}
|
||||
|
||||
fun paginatedNotes(
|
||||
userId: Int,
|
||||
page: Int,
|
||||
itemsPerPage: Int = 20,
|
||||
tag: String? = null,
|
||||
deleted: Boolean = false
|
||||
deleted: Boolean = false,
|
||||
): PaginatedNotes {
|
||||
val count = noteRepository.count(userId, tag, deleted)
|
||||
val offset = (page - 1) * itemsPerPage
|
||||
@ -44,11 +56,38 @@ class NoteService(
|
||||
}
|
||||
|
||||
fun find(userId: Int, uuid: UUID) = noteRepository.find(userId, uuid)
|
||||
fun trash(userId: Int, uuid: UUID) = noteRepository.delete(userId, uuid, permanent = false)
|
||||
fun restore(userId: Int, uuid: UUID) = noteRepository.restore(userId, uuid)
|
||||
fun delete(userId: Int, uuid: UUID) = noteRepository.delete(userId, uuid, permanent = true)
|
||||
|
||||
fun trash(userId: Int, uuid: UUID): Boolean {
|
||||
val res = noteRepository.delete(userId, uuid, permanent = false)
|
||||
if (res) searcher.deleteIndex(userId, uuid)
|
||||
return res
|
||||
}
|
||||
|
||||
fun restore(userId: Int, uuid: UUID): Boolean {
|
||||
val res = noteRepository.restore(userId, uuid)
|
||||
if (res) find(userId, uuid)?.let { note -> searcher.indexNote(userId, note) }
|
||||
return res
|
||||
}
|
||||
|
||||
fun delete(userId: Int, uuid: UUID): Boolean {
|
||||
val res = noteRepository.delete(userId, uuid, permanent = true)
|
||||
if (res) searcher.deleteIndex(userId, uuid)
|
||||
return res
|
||||
}
|
||||
|
||||
fun countDeleted(userId: Int) = noteRepository.count(userId, deleted = true)
|
||||
|
||||
fun indexAll() {
|
||||
val userIds = userRepository.findAll()
|
||||
userIds.forEach { id ->
|
||||
val notes = noteRepository.findAllDetails(id)
|
||||
searcher.indexNotes(id, notes)
|
||||
}
|
||||
}
|
||||
|
||||
fun search(userId: Int, searchTerms: SearchTerms) = searcher.search(userId, searchTerms)
|
||||
|
||||
fun dropAllIndexes() = searcher.dropAll()
|
||||
}
|
||||
|
||||
data class PaginatedNotes(val pages: Int, val notes: List<PersistedNoteMetadata>)
|
||||
|
||||
@ -27,4 +27,5 @@ interface NoteRepository {
|
||||
fun find(userId: Int, uuid: UUID): PersistedNote?
|
||||
fun update(userId: Int, uuid: UUID, note: Note): PersistedNote?
|
||||
fun export(userId: Int): List<ExportedNote>
|
||||
fun findAllDetails(userId: Int): List<PersistedNote>
|
||||
}
|
||||
|
||||
@ -10,4 +10,5 @@ interface UserRepository {
|
||||
fun exists(username: String): Boolean
|
||||
fun exists(id: Int): Boolean
|
||||
fun delete(id: Int): Boolean
|
||||
fun findAll(): List<Int>
|
||||
}
|
||||
|
||||
17
domain/src/main/kotlin/usecases/search/SearchUseCase.kt
Normal file
17
domain/src/main/kotlin/usecases/search/SearchUseCase.kt
Normal file
@ -0,0 +1,17 @@
|
||||
package be.simplenotes.domain.usecases.search
|
||||
|
||||
import be.simplenotes.domain.model.PersistedNote
|
||||
import be.simplenotes.domain.model.PersistedNoteMetadata
|
||||
import java.util.*
|
||||
|
||||
data class SearchTerms(val title: String?, val tag: String?, val content: String?)
|
||||
|
||||
interface NoteSearcher {
|
||||
fun indexNote(userId: Int, note: PersistedNote)
|
||||
fun indexNotes(userId: Int, notes: List<PersistedNote>)
|
||||
fun deleteIndex(userId: Int, uuid: UUID)
|
||||
fun updateIndex(userId: Int, note: PersistedNote)
|
||||
fun search(userId: Int, terms: SearchTerms): List<PersistedNoteMetadata>
|
||||
fun dropIndex(userId: Int)
|
||||
fun dropAll()
|
||||
}
|
||||
@ -197,4 +197,25 @@ internal class NoteRepositoryImpl(private val db: Database) : NoteRepository {
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override fun findAllDetails(userId: Int): List<PersistedNote> {
|
||||
val notes = db.notes
|
||||
.filterColumns { it.columns - it.deleted }
|
||||
.filter { (it.userId eq userId) and (it.deleted eq false) }
|
||||
.toList()
|
||||
|
||||
if (notes.isEmpty()) return emptyList()
|
||||
|
||||
val uuids = notes.map { note -> note.uuid }
|
||||
|
||||
val tagsByUuid = db.tags
|
||||
.filterColumns { listOf(it.noteUuid, it.name) }
|
||||
.filter { it.noteUuid inList uuids }
|
||||
.groupByTo(HashMap(), { it.note.uuid }, { it.name })
|
||||
|
||||
return notes.map { note ->
|
||||
val tags = tagsByUuid[note.uuid] ?: emptyList()
|
||||
note.toPersistedNote(tags)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -28,4 +28,5 @@ internal class UserRepositoryImpl(private val db: Database) : UserRepository {
|
||||
override fun exists(username: String) = db.users.any { it.username eq username }
|
||||
override fun exists(id: Int) = db.users.any { it.id eq id }
|
||||
override fun delete(id: Int) = db.useTransaction { db.delete(Users) { it.id eq id } == 1 }
|
||||
override fun findAll() = db.from(Users).select(Users.id).map { it[Users.id]!! }
|
||||
}
|
||||
|
||||
@ -6,11 +6,11 @@
|
||||
</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
<root level="INFO">
|
||||
<root level="DEBUG">
|
||||
<appender-ref ref="STDOUT"/>
|
||||
</root>
|
||||
<logger name="org.eclipse.jetty" level="INFO"/>
|
||||
<logger name="me.liuwj.ktorm.database" level="DEBUG"/>
|
||||
<logger name="me.liuwj.ktorm.database" level="INFO"/>
|
||||
<logger name="com.zaxxer.hikari" level="INFO"/>
|
||||
<logger name="org.flywaydb.core" level="INFO"/>
|
||||
</configuration>
|
||||
|
||||
1
pom.xml
1
pom.xml
@ -10,6 +10,7 @@
|
||||
<module>app</module>
|
||||
<module>domain</module>
|
||||
<module>shared</module>
|
||||
<module>search</module>
|
||||
</modules>
|
||||
|
||||
<packaging>pom</packaging>
|
||||
|
||||
50
search/pom.xml
Normal file
50
search/pom.xml
Normal file
@ -0,0 +1,50 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>parent</artifactId>
|
||||
<groupId>be.simplenotes</groupId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>search</artifactId>
|
||||
|
||||
<properties>
|
||||
<lucene.version>8.5.2</lucene.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>be.simplenotes</groupId>
|
||||
<artifactId>domain</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-core</artifactId>
|
||||
<version>${lucene.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-analyzers-common</artifactId>
|
||||
<version>${lucene.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-queryparser</artifactId>
|
||||
<version>${lucene.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>be.simplenotes</groupId>
|
||||
<artifactId>shared</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
</project>
|
||||
7
search/src/main/kotlin/Constants.kt
Normal file
7
search/src/main/kotlin/Constants.kt
Normal file
@ -0,0 +1,7 @@
|
||||
package be.simplenotes.search
|
||||
|
||||
internal const val uuidField = "uuid"
|
||||
internal const val titleField = "title"
|
||||
internal const val tagsField = "tags"
|
||||
internal const val contentField = "content"
|
||||
internal const val updatedAtField = "updatedAt"
|
||||
35
search/src/main/kotlin/Extractors.kt
Normal file
35
search/src/main/kotlin/Extractors.kt
Normal file
@ -0,0 +1,35 @@
|
||||
package be.simplenotes.search
|
||||
|
||||
import be.simplenotes.domain.model.PersistedNote
|
||||
import be.simplenotes.domain.model.PersistedNoteMetadata
|
||||
import org.apache.lucene.document.Document
|
||||
import org.apache.lucene.document.Field
|
||||
import org.apache.lucene.document.StringField
|
||||
import org.apache.lucene.document.TextField
|
||||
import org.apache.lucene.search.IndexSearcher
|
||||
import org.apache.lucene.search.TopDocs
|
||||
|
||||
internal fun PersistedNote.toDocument(): Document {
|
||||
val note = this
|
||||
return Document().apply {
|
||||
// non searchable fields
|
||||
add(StringField(uuidField, UuidFieldConverter.toDoc(note.uuid), Field.Store.YES))
|
||||
add(StringField(updatedAtField, LocalDateTimeFieldConverter.toDoc(note.updatedAt), Field.Store.YES))
|
||||
|
||||
// searchable fields
|
||||
add(TextField(titleField, note.meta.title, Field.Store.YES))
|
||||
add(TextField(tagsField, TagsFieldConverter.toDoc(note.meta.tags), Field.Store.YES))
|
||||
add(TextField(contentField, note.html, Field.Store.YES))
|
||||
}
|
||||
}
|
||||
|
||||
internal fun TopDocs.toResults(searcher: IndexSearcher) = scoreDocs.map {
|
||||
searcher.doc(it.doc).let { doc ->
|
||||
PersistedNoteMetadata(
|
||||
title = doc.get(titleField),
|
||||
uuid = UuidFieldConverter.fromDoc(doc.get(uuidField)),
|
||||
updatedAt = LocalDateTimeFieldConverter.fromDoc(doc.get(updatedAtField)),
|
||||
tags = TagsFieldConverter.fromDoc(doc.get(tagsField))
|
||||
)
|
||||
}
|
||||
}
|
||||
26
search/src/main/kotlin/FieldConverters.kt
Normal file
26
search/src/main/kotlin/FieldConverters.kt
Normal file
@ -0,0 +1,26 @@
|
||||
package be.simplenotes.search
|
||||
|
||||
import java.time.LocalDateTime
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.util.*
|
||||
|
||||
internal interface FieldConverter<T> {
|
||||
fun toDoc(value: T): String
|
||||
fun fromDoc(value: String): T
|
||||
}
|
||||
|
||||
internal object LocalDateTimeFieldConverter : FieldConverter<LocalDateTime> {
|
||||
private val formatter = DateTimeFormatter.ISO_DATE_TIME
|
||||
override fun toDoc(value: LocalDateTime): String = formatter.format(value)
|
||||
override fun fromDoc(value: String): LocalDateTime = LocalDateTime.parse(value, formatter)
|
||||
}
|
||||
|
||||
internal object UuidFieldConverter : FieldConverter<UUID> {
|
||||
override fun toDoc(value: UUID): String = value.toString()
|
||||
override fun fromDoc(value: String): UUID = UUID.fromString(value)
|
||||
}
|
||||
|
||||
internal object TagsFieldConverter : FieldConverter<List<String>> {
|
||||
override fun toDoc(value: List<String>): String = value.joinToString(" ")
|
||||
override fun fromDoc(value: String): List<String> = value.split(" ")
|
||||
}
|
||||
117
search/src/main/kotlin/NoteSearcherImpl.kt
Normal file
117
search/src/main/kotlin/NoteSearcherImpl.kt
Normal file
@ -0,0 +1,117 @@
|
||||
package be.simplenotes.search
|
||||
|
||||
import be.simplenotes.domain.model.PersistedNote
|
||||
import be.simplenotes.domain.model.PersistedNoteMetadata
|
||||
import be.simplenotes.domain.usecases.search.NoteSearcher
|
||||
import be.simplenotes.domain.usecases.search.SearchTerms
|
||||
import be.simplenotes.search.utils.rmdir
|
||||
import org.apache.lucene.analysis.standard.StandardAnalyzer
|
||||
import org.apache.lucene.index.*
|
||||
import org.apache.lucene.search.*
|
||||
import org.apache.lucene.store.Directory
|
||||
import org.apache.lucene.store.FSDirectory
|
||||
import org.slf4j.LoggerFactory
|
||||
import java.io.File
|
||||
import java.nio.file.Path
|
||||
import java.util.*
|
||||
|
||||
class NoteSearcherImpl(basePath: Path = Path.of("/tmp", "lucene")) : NoteSearcher {
|
||||
private val baseFile = basePath.toFile()
|
||||
|
||||
private val logger = LoggerFactory.getLogger(javaClass)
|
||||
|
||||
// region utils
|
||||
private fun getDirectory(userId: Int): Directory {
|
||||
val index = File(baseFile, userId.toString()).toPath()
|
||||
return FSDirectory.open(index)
|
||||
}
|
||||
|
||||
private fun getIndexSearcher(userId: Int): IndexSearcher {
|
||||
val directory = getDirectory(userId)
|
||||
val reader: IndexReader = DirectoryReader.open(directory)
|
||||
return IndexSearcher(reader)
|
||||
}
|
||||
// endregion
|
||||
|
||||
override fun indexNote(userId: Int, note: PersistedNote) {
|
||||
logger.debug("Indexing note ${note.uuid} for user $userId")
|
||||
|
||||
val dir = getDirectory(userId)
|
||||
val config = IndexWriterConfig(StandardAnalyzer())
|
||||
val writer = IndexWriter(dir, config)
|
||||
val doc = note.toDocument()
|
||||
|
||||
with(writer) {
|
||||
addDocument(doc)
|
||||
commit()
|
||||
close()
|
||||
}
|
||||
}
|
||||
|
||||
override fun indexNotes(userId: Int, notes: List<PersistedNote>) {
|
||||
logger.debug("Indexing notes for user $userId")
|
||||
|
||||
val dir = getDirectory(userId)
|
||||
val config = IndexWriterConfig(StandardAnalyzer())
|
||||
val writer = IndexWriter(dir, config)
|
||||
val docs = notes.map { it.toDocument() }
|
||||
|
||||
with(writer) {
|
||||
addDocuments(docs)
|
||||
commit()
|
||||
close()
|
||||
}
|
||||
}
|
||||
|
||||
override fun deleteIndex(userId: Int, uuid: UUID) {
|
||||
logger.debug("Deleting index $uuid for user $userId")
|
||||
|
||||
val dir = getDirectory(userId)
|
||||
val config = IndexWriterConfig(StandardAnalyzer())
|
||||
val writer = IndexWriter(dir, config)
|
||||
|
||||
with(writer) {
|
||||
deleteDocuments(TermQuery(Term(uuidField, UuidFieldConverter.toDoc(uuid))))
|
||||
commit()
|
||||
close()
|
||||
}
|
||||
}
|
||||
|
||||
override fun updateIndex(userId: Int, note: PersistedNote) {
|
||||
logger.debug("Updating note ${note.uuid} for user $userId")
|
||||
deleteIndex(userId, note.uuid)
|
||||
indexNote(userId, note)
|
||||
}
|
||||
|
||||
override fun search(userId: Int, terms: SearchTerms): List<PersistedNoteMetadata> {
|
||||
val searcher = getIndexSearcher(userId)
|
||||
|
||||
val builder = BooleanQuery.Builder()
|
||||
|
||||
terms.title?.let {
|
||||
val titleQuery = FuzzyQuery(Term(titleField, it))
|
||||
builder.add(BooleanClause(titleQuery, BooleanClause.Occur.SHOULD))
|
||||
}
|
||||
|
||||
terms.tag?.let {
|
||||
val tagsQuery = FuzzyQuery(Term(tagsField, it))
|
||||
builder.add(BooleanClause(tagsQuery, BooleanClause.Occur.SHOULD))
|
||||
}
|
||||
|
||||
terms.content?.let {
|
||||
val contentQuery = FuzzyQuery(Term(contentField, it))
|
||||
builder.add(BooleanClause(contentQuery, BooleanClause.Occur.SHOULD))
|
||||
}
|
||||
|
||||
val query = builder.build()
|
||||
|
||||
val topDocs = searcher.search(query, 10)
|
||||
logger.debug("Searching: `$query` results: ${topDocs.totalHits.value}")
|
||||
return topDocs.toResults(searcher)
|
||||
}
|
||||
|
||||
override fun dropIndex(userId: Int) = rmdir(File(baseFile, userId.toString()).toPath())
|
||||
|
||||
override fun dropAll() = rmdir(baseFile.toPath())
|
||||
|
||||
}
|
||||
9
search/src/main/kotlin/SeachModule.kt
Normal file
9
search/src/main/kotlin/SeachModule.kt
Normal file
@ -0,0 +1,9 @@
|
||||
package be.simplenotes.search
|
||||
|
||||
import be.simplenotes.domain.usecases.search.NoteSearcher
|
||||
import org.koin.dsl.module
|
||||
import java.nio.file.Path
|
||||
|
||||
val searchModule = module {
|
||||
single<NoteSearcher> { NoteSearcherImpl(Path.of(".lucene")) }
|
||||
}
|
||||
29
search/src/main/kotlin/utils/PathUtils.kt
Normal file
29
search/src/main/kotlin/utils/PathUtils.kt
Normal file
@ -0,0 +1,29 @@
|
||||
package be.simplenotes.search.utils
|
||||
|
||||
import java.io.IOException
|
||||
import java.nio.file.FileVisitResult
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.SimpleFileVisitor
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
|
||||
internal fun rmdir(path: Path) {
|
||||
try {
|
||||
Files.walkFileTree(
|
||||
path,
|
||||
object : SimpleFileVisitor<Path>() {
|
||||
override fun visitFile(file: Path, attrs: BasicFileAttributes?): FileVisitResult {
|
||||
Files.delete(file)
|
||||
return FileVisitResult.CONTINUE
|
||||
}
|
||||
|
||||
override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult {
|
||||
Files.delete(dir)
|
||||
return FileVisitResult.CONTINUE
|
||||
}
|
||||
}
|
||||
)
|
||||
} catch (e: IOException) {
|
||||
// This is fine
|
||||
}
|
||||
}
|
||||
163
search/src/test/kotlin/NoteSearcherImplTest.kt
Normal file
163
search/src/test/kotlin/NoteSearcherImplTest.kt
Normal file
@ -0,0 +1,163 @@
|
||||
package be.simplenotes.search
|
||||
|
||||
import be.simplenotes.domain.model.NoteMetadata
|
||||
import be.simplenotes.domain.model.PersistedNote
|
||||
import be.simplenotes.domain.model.PersistedNoteMetadata
|
||||
import be.simplenotes.domain.usecases.search.SearchTerms
|
||||
import org.assertj.core.api.Assertions.assertThat
|
||||
import org.intellij.lang.annotations.Language
|
||||
import org.junit.jupiter.api.AfterAll
|
||||
import org.junit.jupiter.api.BeforeEach
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.api.parallel.ResourceLock
|
||||
import java.time.LocalDateTime
|
||||
import java.util.*
|
||||
|
||||
@ResourceLock("lucene")
|
||||
internal class NoteSearcherImplTest {
|
||||
|
||||
// region setup
|
||||
private val searcher = NoteSearcherImpl()
|
||||
|
||||
private fun index(
|
||||
title: String,
|
||||
tags: List<String> = emptyList(),
|
||||
content: String = "",
|
||||
uuid: UUID = UUID.randomUUID(),
|
||||
): PersistedNote {
|
||||
val note = PersistedNote(NoteMetadata(title, tags), markdown = "", content, LocalDateTime.now(), uuid)
|
||||
searcher.indexNote(1, note)
|
||||
return note
|
||||
}
|
||||
|
||||
private fun search(
|
||||
title: String? = null,
|
||||
tag: String? = null,
|
||||
content: String? = null,
|
||||
): List<PersistedNoteMetadata> = searcher.search(1, SearchTerms(title, tag, content))
|
||||
|
||||
@BeforeEach
|
||||
@AfterAll
|
||||
fun dropIndexes() {
|
||||
searcher.dropIndex(1)
|
||||
}
|
||||
// endregion
|
||||
|
||||
@Test
|
||||
fun `exact title search`() {
|
||||
index("first")
|
||||
index("second")
|
||||
index("flip")
|
||||
|
||||
assertThat(search("first"))
|
||||
.hasSizeGreaterThanOrEqualTo(1)
|
||||
.anyMatch { it.title == "first" }
|
||||
|
||||
assertThat(search("nothing")).isEmpty()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `fuzzy title search`() {
|
||||
index("first")
|
||||
index("second")
|
||||
index("flip")
|
||||
|
||||
assertThat(search("firt"))
|
||||
.hasSizeGreaterThanOrEqualTo(1)
|
||||
.anyMatch { it.title == "first" }
|
||||
|
||||
assertThat(search("nothing")).isEmpty()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `exact tags search`() {
|
||||
index("first", tags = listOf("example", "flamingo"))
|
||||
index("second", tags = listOf("yes"))
|
||||
index("second")
|
||||
|
||||
assertThat(search(tag = "example"))
|
||||
.hasSize(1)
|
||||
.anyMatch { it.title == "first" }
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `exact content search`() {
|
||||
@Language("html")
|
||||
val content =
|
||||
"""
|
||||
<div>
|
||||
<h1 class="title">Apache Lucene Core</h1>
|
||||
<p>Apache Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> is a
|
||||
high-performance, full-featured text search engine library written entirely in Java.
|
||||
It is a technology suitable for nearly any application that requires full-text search,
|
||||
especially cross-platform.</p>
|
||||
<p>Apache Lucene is an open source project available for free download. Please use the
|
||||
links on the right to access Lucene.</p>
|
||||
<h1 id="lucenetm-features">Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> Features</h1>
|
||||
<p>Lucene offers powerful features through a simple API:</p>
|
||||
<h2 id="scalable-high-performance-indexing">Scalable, High-Performance Indexing</h2>
|
||||
<ul>
|
||||
<li>over <a href="http://home.apache.org/~mikemccand/lucenebench/indexing.html">150GB/hour on modern hardware</a></li>
|
||||
<li>small RAM requirements -- only 1MB heap</li>
|
||||
<li>incremental indexing as fast as batch indexing</li>
|
||||
<li>index size roughly 20-30% the size of text indexed</li>
|
||||
</ul>
|
||||
""".trimIndent()
|
||||
|
||||
index("first", content = content)
|
||||
|
||||
assertThat(search(content = "fast"))
|
||||
.hasSize(1)
|
||||
.anyMatch { it.title == "first" }
|
||||
|
||||
@Suppress("SpellCheckingInspection")
|
||||
assertThat(search(content = "preformance")) // <- note the error
|
||||
.hasSize(1)
|
||||
.anyMatch { it.title == "first" }
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `combined search`() {
|
||||
@Language("html")
|
||||
val content =
|
||||
"""
|
||||
<div>
|
||||
<h1 class="title">Apache Lucene Core</h1>
|
||||
<p>Apache Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> is a
|
||||
high-performance, full-featured text search engine library written entirely in Java.
|
||||
It is a technology suitable for nearly any application that requires full-text search,
|
||||
especially cross-platform.</p>
|
||||
<p>Apache Lucene is an open source project available for free download. Please use the
|
||||
links on the right to access Lucene.</p>
|
||||
<h1 id="lucenetm-features">Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> Features</h1>
|
||||
<p>Lucene offers powerful features through a simple API:</p>
|
||||
<h2 id="scalable-high-performance-indexing">Scalable, High-Performance Indexing</h2>
|
||||
<ul>
|
||||
<li>over <a href="http://home.apache.org/~mikemccand/lucenebench/indexing.html">150GB/hour on modern hardware</a></li>
|
||||
<li>small RAM requirements -- only 1MB heap</li>
|
||||
<li>incremental indexing as fast as batch indexing</li>
|
||||
<li>index size roughly 20-30% the size of text indexed</li>
|
||||
</ul>
|
||||
""".trimIndent()
|
||||
|
||||
index("first", content = content, tags = listOf("abc"))
|
||||
|
||||
assertThat(search(title = "fir", tag = "abc", content = "20"))
|
||||
.hasSize(1)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `delete index`() {
|
||||
val uuid = index("first").uuid
|
||||
searcher.deleteIndex(1, uuid)
|
||||
assertThat(search("first")).isEmpty()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `update index`() {
|
||||
val note = index("first")
|
||||
searcher.updateIndex(1, note.copy(meta = note.meta.copy(title = "new")))
|
||||
assertThat(search("first")).isEmpty()
|
||||
assertThat(search("new")).hasSize(1)
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user