Merge branch 'lucene-search'

This commit is contained in:
Hubert Van De Walle 2020-08-19 19:32:37 +02:00
commit 7305fb47c7
27 changed files with 684 additions and 29 deletions

3
.gitignore vendored
View File

@ -130,3 +130,6 @@ app/src/main/resources/static/styles*
# h2 db
*.db
# lucene index
.lucene/

View File

@ -8,6 +8,7 @@ COPY app/pom.xml app/pom.xml
COPY domain/pom.xml domain/pom.xml
COPY persistance/pom.xml persistance/pom.xml
COPY shared/pom.xml shared/pom.xml
COPY search/pom.xml search/pom.xml
RUN mvn verify clean --fail-never
@ -15,6 +16,7 @@ COPY app/src app/src
COPY domain/src domain/src
COPY persistance/src persistance/src
COPY shared/src shared/src
COPY search/src search/src
RUN mvn -Dstyle.color=always package

View File

@ -14,6 +14,11 @@
<artifactId>persistance</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>be.simplenotes</groupId>
<artifactId>search</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>be.simplenotes</groupId>
<artifactId>domain</artifactId>
@ -104,6 +109,12 @@
<include>**</include>
</includes>
</filter>
<filter>
<artifact>org.apache.lucene:*</artifact>
<includes>
<include>**</include>
</includes>
</filter>
<filter>
<artifact>*:*</artifact>
<excludes>

View File

@ -14,8 +14,10 @@ import be.simplenotes.app.views.NoteView
import be.simplenotes.app.views.SettingView
import be.simplenotes.app.views.UserView
import be.simplenotes.domain.domainModule
import be.simplenotes.domain.usecases.NoteService
import be.simplenotes.persistance.DbMigrations
import be.simplenotes.persistance.persistanceModule
import be.simplenotes.search.searchModule
import be.simplenotes.shared.config.DataSourceConfig
import be.simplenotes.shared.config.JwtConfig
import org.http4k.core.RequestContexts
@ -36,6 +38,7 @@ fun main() {
baseModule,
noteModule,
settingsModule,
searchModule,
)
}.koin
@ -50,6 +53,10 @@ fun main() {
val migrations = koin.get<DbMigrations>()
migrations.migrate()
val noteService = koin.get<NoteService>()
noteService.dropAllIndexes()
noteService.indexAll()
koin.get<Server>().start()
}

View File

@ -2,12 +2,14 @@ package be.simplenotes.app.controllers
import be.simplenotes.app.extensions.html
import be.simplenotes.app.extensions.redirect
import be.simplenotes.app.utils.parseSearchTerms
import be.simplenotes.app.views.NoteView
import be.simplenotes.domain.security.JwtPayload
import be.simplenotes.domain.usecases.NoteService
import be.simplenotes.domain.usecases.markdown.InvalidMeta
import be.simplenotes.domain.usecases.markdown.MissingMeta
import be.simplenotes.domain.usecases.markdown.ValidationError
import be.simplenotes.domain.usecases.search.SearchTerms
import org.http4k.core.Method
import org.http4k.core.Request
import org.http4k.core.Response
@ -34,7 +36,9 @@ class NoteController(
val html = when (it) {
MissingMeta -> view.noteEditor(jwtPayload, error = "Missing note metadata", textarea = markdownForm)
InvalidMeta -> view.noteEditor(jwtPayload, error = "Invalid note metadata", textarea = markdownForm)
is ValidationError -> view.noteEditor(jwtPayload, validationErrors = it.validationErrors, textarea = markdownForm)
is ValidationError -> view.noteEditor(jwtPayload,
validationErrors = it.validationErrors,
textarea = markdownForm)
}
Response(BAD_REQUEST).html(html)
},
@ -52,6 +56,13 @@ class NoteController(
return Response(OK).html(view.notes(jwtPayload, notes, currentPage, pages, deletedCount, tag = tag))
}
fun search(request: Request, jwtPayload: JwtPayload): Response {
val terms = request.searchTerms()
val notes = noteService.search(jwtPayload.userId, terms)
val deletedCount = noteService.countDeleted(jwtPayload.userId)
return Response(OK).html(view.search(jwtPayload, notes, deletedCount))
}
fun note(request: Request, jwtPayload: JwtPayload): Response {
val noteUuid = request.uuidPath() ?: return Response(NOT_FOUND)
@ -121,4 +132,6 @@ class NoteController(
null
}
}
private fun Request.searchTerms(): SearchTerms = parseSearchTerms(form("search") ?: "")
}

View File

@ -48,6 +48,7 @@ class Router(
"/settings" bind POST to { protected(it, settingsController::settings) },
"/export" bind POST to { protected(it, settingsController::export) },
"/notes" bind GET to { protected(it, noteController::list) },
"/notes" bind POST to { protected(it, noteController::search) },
"/notes/new" bind GET to { protected(it, noteController::new) },
"/notes/new" bind POST to { protected(it, noteController::new) },
"/notes/trash" bind GET to { protected(it, noteController::trash) },

View File

@ -0,0 +1,33 @@
package be.simplenotes.app.utils
import be.simplenotes.domain.usecases.search.SearchTerms
private val titleRe = """title:['"](?<title>.*?)['"]""".toRegex()
private val outerTitleRe = """(?<title>title:['"].*?['"])""".toRegex()
private val tagRe = """tag:['"](?<tag>.*?)['"]""".toRegex()
private val outerTagRe = """(?<tag>tag:['"].*?['"])""".toRegex()
fun parseSearchTerms(input: String): SearchTerms {
val title: String? = titleRe.find(input)?.groups?.get(1)?.value
val tag: String? = tagRe.find(input)?.groups?.get(1)?.value
var c: String = input
if (title != null) {
val titleGroup = outerTitleRe.find(input)?.groups?.get(1)?.value
titleGroup?.let { c = c.replace(it, "") }
}
if (tag != null) {
val tagGroup = outerTagRe.find(input)?.groups?.get(1)?.value
tagGroup?.let { c = c.replace(it, "") }
}
val content = c.trim().ifEmpty { null }
return SearchTerms(
title = title,
tag = tag,
content = content
)
}

View File

@ -55,20 +55,7 @@ class NoteView(staticFileResolver: StaticFileResolver) : View(staticFileResolver
tag: String?,
) = renderPage(title = "Notes", jwtPayload = jwtPayload) {
div("container mx-auto p-4") {
div("flex justify-between mb-4") {
h1("text-2xl underline") { +"Notes" }
span {
a(
href = "/notes/trash",
classes = "underline font-semibold"
) { +"Trash ($numberOfDeletedNotes)" }
a(
href = "/notes/new",
classes = "ml-2 btn btn-green"
) { +"New" }
}
}
noteListHeader(numberOfDeletedNotes)
if (notes.isNotEmpty())
noteTable(notes)
else
@ -81,11 +68,22 @@ class NoteView(staticFileResolver: StaticFileResolver) : View(staticFileResolver
}
}
fun search(
jwtPayload: JwtPayload,
notes: List<PersistedNoteMetadata>,
numberOfDeletedNotes: Int,
) = renderPage("Notes", jwtPayload = jwtPayload) {
div("container mx-auto p-4") {
noteListHeader(numberOfDeletedNotes)
noteTable(notes)
}
}
fun trash(
jwtPayload: JwtPayload,
notes: List<PersistedNoteMetadata>,
currentPage: Int,
numberOfPages: Int
numberOfPages: Int,
) = renderPage(title = "Notes", jwtPayload = jwtPayload) {
div("container mx-auto p-4") {
div("flex justify-between mb-4") {

View File

@ -0,0 +1,31 @@
package be.simplenotes.app.views.components
import kotlinx.html.*
import kotlinx.html.ButtonType.submit
import kotlinx.html.FormMethod.post
fun DIV.noteListHeader(numberOfDeletedNotes: Int) {
div("flex justify-between mb-4") {
h1("text-2xl underline") { +"Notes" }
span {
a(
href = "/notes/trash",
classes = "underline font-semibold"
) { +"Trash ($numberOfDeletedNotes)" }
a(
href = "/notes/new",
classes = "ml-2 btn btn-green"
) { +"New" }
}
}
form(method = post, classes = "mb-4") {
val colors = "bg-gray-800 border-gray-700 focus:border-teal-500 text-white"
input(
name = "search",
classes = "$colors rounded w-3/4 border appearance-none focus:outline-none text-base p-2"
)
button(type = submit, classes = "btn btn-green w-1/4") {
+"search"
}
}
}

View File

@ -0,0 +1,39 @@
package be.simplenotes.app.utils
import be.simplenotes.domain.usecases.search.SearchTerms
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import java.util.stream.Stream
internal class SearchTermsParserKtTest {
private fun createResult(
input: String,
title: String? = null,
tag: String? = null,
content: String? = null,
): Pair<String, SearchTerms> = input to SearchTerms(title, tag, content)
@Suppress("Unused")
private fun results() = Stream.of(
createResult("title:'example'", title = "example"),
createResult("title:'example with words'", title = "example with words"),
createResult("title:'example with words'", title = "example with words"),
createResult("""title:"double quotes"""", title = "double quotes"),
createResult("title:'example' something else", title = "example", content = "something else"),
createResult("tag:'example'", tag = "example"),
createResult("tag:'example' title:'other'", title = "other", tag = "example"),
createResult("blah blah tag:'example' title:'other'", title = "other", tag = "example", content = "blah blah"),
createResult("tag:'example' middle title:'other'", title = "other", tag = "example", content = "middle"),
createResult("tag:'example' title:'other' end", title = "other", tag = "example", content = "end"),
createResult("tag:'example abc' title:'other with words' this is the end ", title = "other with words", tag = "example abc", content = "this is the end"),
)
@ParameterizedTest
@MethodSource("results")
fun `valid search parser`(case: Pair<String, SearchTerms>) {
assertThat(parseSearchTerms(case.first)).isEqualTo(case.second)
}
}

View File

@ -26,7 +26,7 @@ val domainModule = module {
single<PasswordHash> { BcryptPasswordHash() }
single { SimpleJwt(get()) }
single { JwtPayloadExtractor(get()) }
single { NoteService(get(), get()) }
single { NoteService(get(), get(), get(), get()) }
single<MarkdownConverter> { MarkdownConverterImpl() }
single<ExportUseCase> { ExportUseCaseImpl(get()) }
}

View File

@ -1,6 +1,7 @@
package be.simplenotes.domain.usecases
import arrow.core.Either
import arrow.core.extensions.fx
import be.simplenotes.domain.model.Note
import be.simplenotes.domain.model.PersistedNote
import be.simplenotes.domain.model.PersistedNoteMetadata
@ -8,33 +9,44 @@ import be.simplenotes.domain.security.HtmlSanitizer
import be.simplenotes.domain.usecases.markdown.MarkdownConverter
import be.simplenotes.domain.usecases.markdown.MarkdownParsingError
import be.simplenotes.domain.usecases.repositories.NoteRepository
import be.simplenotes.domain.usecases.repositories.UserRepository
import be.simplenotes.domain.usecases.search.NoteSearcher
import be.simplenotes.domain.usecases.search.SearchTerms
import java.util.*
class NoteService(
private val markdownConverter: MarkdownConverter,
private val noteRepository: NoteRepository,
private val userRepository: UserRepository,
private val searcher: NoteSearcher,
) {
fun create(userId: Int, markdownText: String): Either<MarkdownParsingError, PersistedNote> =
markdownConverter
.renderDocument(markdownText)
fun create(userId: Int, markdownText: String) = Either.fx<MarkdownParsingError, PersistedNote> {
val persistedNote = !markdownConverter.renderDocument(markdownText)
.map { it.copy(html = HtmlSanitizer.sanitize(it.html)) }
.map { Note(it.metadata, markdown = markdownText, html = it.html) }
.map { noteRepository.create(userId, it) }
fun update(userId: Int, uuid: UUID, markdownText: String): Either<MarkdownParsingError, PersistedNote?> =
markdownConverter
.renderDocument(markdownText)
searcher.indexNote(userId, persistedNote)
persistedNote
}
fun update(userId: Int, uuid: UUID, markdownText: String) = Either.fx<MarkdownParsingError, PersistedNote?> {
val persistedNote = !markdownConverter.renderDocument(markdownText)
.map { it.copy(html = HtmlSanitizer.sanitize(it.html)) }
.map { Note(it.metadata, markdown = markdownText, html = it.html) }
.map { noteRepository.update(userId, uuid, it) }
persistedNote?.let { searcher.updateIndex(userId, it) }
persistedNote
}
fun paginatedNotes(
userId: Int,
page: Int,
itemsPerPage: Int = 20,
tag: String? = null,
deleted: Boolean = false
deleted: Boolean = false,
): PaginatedNotes {
val count = noteRepository.count(userId, tag, deleted)
val offset = (page - 1) * itemsPerPage
@ -44,11 +56,38 @@ class NoteService(
}
fun find(userId: Int, uuid: UUID) = noteRepository.find(userId, uuid)
fun trash(userId: Int, uuid: UUID) = noteRepository.delete(userId, uuid, permanent = false)
fun restore(userId: Int, uuid: UUID) = noteRepository.restore(userId, uuid)
fun delete(userId: Int, uuid: UUID) = noteRepository.delete(userId, uuid, permanent = true)
fun trash(userId: Int, uuid: UUID): Boolean {
val res = noteRepository.delete(userId, uuid, permanent = false)
if (res) searcher.deleteIndex(userId, uuid)
return res
}
fun restore(userId: Int, uuid: UUID): Boolean {
val res = noteRepository.restore(userId, uuid)
if (res) find(userId, uuid)?.let { note -> searcher.indexNote(userId, note) }
return res
}
fun delete(userId: Int, uuid: UUID): Boolean {
val res = noteRepository.delete(userId, uuid, permanent = true)
if (res) searcher.deleteIndex(userId, uuid)
return res
}
fun countDeleted(userId: Int) = noteRepository.count(userId, deleted = true)
fun indexAll() {
val userIds = userRepository.findAll()
userIds.forEach { id ->
val notes = noteRepository.findAllDetails(id)
searcher.indexNotes(id, notes)
}
}
fun search(userId: Int, searchTerms: SearchTerms) = searcher.search(userId, searchTerms)
fun dropAllIndexes() = searcher.dropAll()
}
data class PaginatedNotes(val pages: Int, val notes: List<PersistedNoteMetadata>)

View File

@ -27,4 +27,5 @@ interface NoteRepository {
fun find(userId: Int, uuid: UUID): PersistedNote?
fun update(userId: Int, uuid: UUID, note: Note): PersistedNote?
fun export(userId: Int): List<ExportedNote>
fun findAllDetails(userId: Int): List<PersistedNote>
}

View File

@ -10,4 +10,5 @@ interface UserRepository {
fun exists(username: String): Boolean
fun exists(id: Int): Boolean
fun delete(id: Int): Boolean
fun findAll(): List<Int>
}

View File

@ -0,0 +1,17 @@
package be.simplenotes.domain.usecases.search
import be.simplenotes.domain.model.PersistedNote
import be.simplenotes.domain.model.PersistedNoteMetadata
import java.util.*
data class SearchTerms(val title: String?, val tag: String?, val content: String?)
interface NoteSearcher {
fun indexNote(userId: Int, note: PersistedNote)
fun indexNotes(userId: Int, notes: List<PersistedNote>)
fun deleteIndex(userId: Int, uuid: UUID)
fun updateIndex(userId: Int, note: PersistedNote)
fun search(userId: Int, terms: SearchTerms): List<PersistedNoteMetadata>
fun dropIndex(userId: Int)
fun dropAll()
}

View File

@ -197,4 +197,25 @@ internal class NoteRepositoryImpl(private val db: Database) : NoteRepository {
)
}
}
override fun findAllDetails(userId: Int): List<PersistedNote> {
val notes = db.notes
.filterColumns { it.columns - it.deleted }
.filter { (it.userId eq userId) and (it.deleted eq false) }
.toList()
if (notes.isEmpty()) return emptyList()
val uuids = notes.map { note -> note.uuid }
val tagsByUuid = db.tags
.filterColumns { listOf(it.noteUuid, it.name) }
.filter { it.noteUuid inList uuids }
.groupByTo(HashMap(), { it.note.uuid }, { it.name })
return notes.map { note ->
val tags = tagsByUuid[note.uuid] ?: emptyList()
note.toPersistedNote(tags)
}
}
}

View File

@ -28,4 +28,5 @@ internal class UserRepositoryImpl(private val db: Database) : UserRepository {
override fun exists(username: String) = db.users.any { it.username eq username }
override fun exists(id: Int) = db.users.any { it.id eq id }
override fun delete(id: Int) = db.useTransaction { db.delete(Users) { it.id eq id } == 1 }
override fun findAll() = db.from(Users).select(Users.id).map { it[Users.id]!! }
}

View File

@ -6,11 +6,11 @@
</pattern>
</encoder>
</appender>
<root level="INFO">
<root level="DEBUG">
<appender-ref ref="STDOUT"/>
</root>
<logger name="org.eclipse.jetty" level="INFO"/>
<logger name="me.liuwj.ktorm.database" level="DEBUG"/>
<logger name="me.liuwj.ktorm.database" level="INFO"/>
<logger name="com.zaxxer.hikari" level="INFO"/>
<logger name="org.flywaydb.core" level="INFO"/>
</configuration>

View File

@ -10,6 +10,7 @@
<module>app</module>
<module>domain</module>
<module>shared</module>
<module>search</module>
</modules>
<packaging>pom</packaging>

50
search/pom.xml Normal file
View File

@ -0,0 +1,50 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>parent</artifactId>
<groupId>be.simplenotes</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>search</artifactId>
<properties>
<lucene.version>8.5.2</lucene.version>
</properties>
<dependencies>
<dependency>
<groupId>be.simplenotes</groupId>
<artifactId>domain</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>be.simplenotes</groupId>
<artifactId>shared</artifactId>
<version>1.0-SNAPSHOT</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,7 @@
package be.simplenotes.search
internal const val uuidField = "uuid"
internal const val titleField = "title"
internal const val tagsField = "tags"
internal const val contentField = "content"
internal const val updatedAtField = "updatedAt"

View File

@ -0,0 +1,35 @@
package be.simplenotes.search
import be.simplenotes.domain.model.PersistedNote
import be.simplenotes.domain.model.PersistedNoteMetadata
import org.apache.lucene.document.Document
import org.apache.lucene.document.Field
import org.apache.lucene.document.StringField
import org.apache.lucene.document.TextField
import org.apache.lucene.search.IndexSearcher
import org.apache.lucene.search.TopDocs
internal fun PersistedNote.toDocument(): Document {
val note = this
return Document().apply {
// non searchable fields
add(StringField(uuidField, UuidFieldConverter.toDoc(note.uuid), Field.Store.YES))
add(StringField(updatedAtField, LocalDateTimeFieldConverter.toDoc(note.updatedAt), Field.Store.YES))
// searchable fields
add(TextField(titleField, note.meta.title, Field.Store.YES))
add(TextField(tagsField, TagsFieldConverter.toDoc(note.meta.tags), Field.Store.YES))
add(TextField(contentField, note.html, Field.Store.YES))
}
}
internal fun TopDocs.toResults(searcher: IndexSearcher) = scoreDocs.map {
searcher.doc(it.doc).let { doc ->
PersistedNoteMetadata(
title = doc.get(titleField),
uuid = UuidFieldConverter.fromDoc(doc.get(uuidField)),
updatedAt = LocalDateTimeFieldConverter.fromDoc(doc.get(updatedAtField)),
tags = TagsFieldConverter.fromDoc(doc.get(tagsField))
)
}
}

View File

@ -0,0 +1,26 @@
package be.simplenotes.search
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import java.util.*
internal interface FieldConverter<T> {
fun toDoc(value: T): String
fun fromDoc(value: String): T
}
internal object LocalDateTimeFieldConverter : FieldConverter<LocalDateTime> {
private val formatter = DateTimeFormatter.ISO_DATE_TIME
override fun toDoc(value: LocalDateTime): String = formatter.format(value)
override fun fromDoc(value: String): LocalDateTime = LocalDateTime.parse(value, formatter)
}
internal object UuidFieldConverter : FieldConverter<UUID> {
override fun toDoc(value: UUID): String = value.toString()
override fun fromDoc(value: String): UUID = UUID.fromString(value)
}
internal object TagsFieldConverter : FieldConverter<List<String>> {
override fun toDoc(value: List<String>): String = value.joinToString(" ")
override fun fromDoc(value: String): List<String> = value.split(" ")
}

View File

@ -0,0 +1,117 @@
package be.simplenotes.search
import be.simplenotes.domain.model.PersistedNote
import be.simplenotes.domain.model.PersistedNoteMetadata
import be.simplenotes.domain.usecases.search.NoteSearcher
import be.simplenotes.domain.usecases.search.SearchTerms
import be.simplenotes.search.utils.rmdir
import org.apache.lucene.analysis.standard.StandardAnalyzer
import org.apache.lucene.index.*
import org.apache.lucene.search.*
import org.apache.lucene.store.Directory
import org.apache.lucene.store.FSDirectory
import org.slf4j.LoggerFactory
import java.io.File
import java.nio.file.Path
import java.util.*
class NoteSearcherImpl(basePath: Path = Path.of("/tmp", "lucene")) : NoteSearcher {
private val baseFile = basePath.toFile()
private val logger = LoggerFactory.getLogger(javaClass)
// region utils
private fun getDirectory(userId: Int): Directory {
val index = File(baseFile, userId.toString()).toPath()
return FSDirectory.open(index)
}
private fun getIndexSearcher(userId: Int): IndexSearcher {
val directory = getDirectory(userId)
val reader: IndexReader = DirectoryReader.open(directory)
return IndexSearcher(reader)
}
// endregion
override fun indexNote(userId: Int, note: PersistedNote) {
logger.debug("Indexing note ${note.uuid} for user $userId")
val dir = getDirectory(userId)
val config = IndexWriterConfig(StandardAnalyzer())
val writer = IndexWriter(dir, config)
val doc = note.toDocument()
with(writer) {
addDocument(doc)
commit()
close()
}
}
override fun indexNotes(userId: Int, notes: List<PersistedNote>) {
logger.debug("Indexing notes for user $userId")
val dir = getDirectory(userId)
val config = IndexWriterConfig(StandardAnalyzer())
val writer = IndexWriter(dir, config)
val docs = notes.map { it.toDocument() }
with(writer) {
addDocuments(docs)
commit()
close()
}
}
override fun deleteIndex(userId: Int, uuid: UUID) {
logger.debug("Deleting index $uuid for user $userId")
val dir = getDirectory(userId)
val config = IndexWriterConfig(StandardAnalyzer())
val writer = IndexWriter(dir, config)
with(writer) {
deleteDocuments(TermQuery(Term(uuidField, UuidFieldConverter.toDoc(uuid))))
commit()
close()
}
}
override fun updateIndex(userId: Int, note: PersistedNote) {
logger.debug("Updating note ${note.uuid} for user $userId")
deleteIndex(userId, note.uuid)
indexNote(userId, note)
}
override fun search(userId: Int, terms: SearchTerms): List<PersistedNoteMetadata> {
val searcher = getIndexSearcher(userId)
val builder = BooleanQuery.Builder()
terms.title?.let {
val titleQuery = FuzzyQuery(Term(titleField, it))
builder.add(BooleanClause(titleQuery, BooleanClause.Occur.SHOULD))
}
terms.tag?.let {
val tagsQuery = FuzzyQuery(Term(tagsField, it))
builder.add(BooleanClause(tagsQuery, BooleanClause.Occur.SHOULD))
}
terms.content?.let {
val contentQuery = FuzzyQuery(Term(contentField, it))
builder.add(BooleanClause(contentQuery, BooleanClause.Occur.SHOULD))
}
val query = builder.build()
val topDocs = searcher.search(query, 10)
logger.debug("Searching: `$query` results: ${topDocs.totalHits.value}")
return topDocs.toResults(searcher)
}
override fun dropIndex(userId: Int) = rmdir(File(baseFile, userId.toString()).toPath())
override fun dropAll() = rmdir(baseFile.toPath())
}

View File

@ -0,0 +1,9 @@
package be.simplenotes.search
import be.simplenotes.domain.usecases.search.NoteSearcher
import org.koin.dsl.module
import java.nio.file.Path
val searchModule = module {
single<NoteSearcher> { NoteSearcherImpl(Path.of(".lucene")) }
}

View File

@ -0,0 +1,29 @@
package be.simplenotes.search.utils
import java.io.IOException
import java.nio.file.FileVisitResult
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
internal fun rmdir(path: Path) {
try {
Files.walkFileTree(
path,
object : SimpleFileVisitor<Path>() {
override fun visitFile(file: Path, attrs: BasicFileAttributes?): FileVisitResult {
Files.delete(file)
return FileVisitResult.CONTINUE
}
override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult {
Files.delete(dir)
return FileVisitResult.CONTINUE
}
}
)
} catch (e: IOException) {
// This is fine
}
}

View File

@ -0,0 +1,163 @@
package be.simplenotes.search
import be.simplenotes.domain.model.NoteMetadata
import be.simplenotes.domain.model.PersistedNote
import be.simplenotes.domain.model.PersistedNoteMetadata
import be.simplenotes.domain.usecases.search.SearchTerms
import org.assertj.core.api.Assertions.assertThat
import org.intellij.lang.annotations.Language
import org.junit.jupiter.api.AfterAll
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.parallel.ResourceLock
import java.time.LocalDateTime
import java.util.*
@ResourceLock("lucene")
internal class NoteSearcherImplTest {
// region setup
private val searcher = NoteSearcherImpl()
private fun index(
title: String,
tags: List<String> = emptyList(),
content: String = "",
uuid: UUID = UUID.randomUUID(),
): PersistedNote {
val note = PersistedNote(NoteMetadata(title, tags), markdown = "", content, LocalDateTime.now(), uuid)
searcher.indexNote(1, note)
return note
}
private fun search(
title: String? = null,
tag: String? = null,
content: String? = null,
): List<PersistedNoteMetadata> = searcher.search(1, SearchTerms(title, tag, content))
@BeforeEach
@AfterAll
fun dropIndexes() {
searcher.dropIndex(1)
}
// endregion
@Test
fun `exact title search`() {
index("first")
index("second")
index("flip")
assertThat(search("first"))
.hasSizeGreaterThanOrEqualTo(1)
.anyMatch { it.title == "first" }
assertThat(search("nothing")).isEmpty()
}
@Test
fun `fuzzy title search`() {
index("first")
index("second")
index("flip")
assertThat(search("firt"))
.hasSizeGreaterThanOrEqualTo(1)
.anyMatch { it.title == "first" }
assertThat(search("nothing")).isEmpty()
}
@Test
fun `exact tags search`() {
index("first", tags = listOf("example", "flamingo"))
index("second", tags = listOf("yes"))
index("second")
assertThat(search(tag = "example"))
.hasSize(1)
.anyMatch { it.title == "first" }
}
@Test
fun `exact content search`() {
@Language("html")
val content =
"""
<div>
<h1 class="title">Apache Lucene Core</h1>
<p>Apache Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> is a
high-performance, full-featured text search engine library written entirely in Java.
It is a technology suitable for nearly any application that requires full-text search,
especially cross-platform.</p>
<p>Apache Lucene is an open source project available for free download. Please use the
links on the right to access Lucene.</p>
<h1 id="lucenetm-features">Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> Features</h1>
<p>Lucene offers powerful features through a simple API:</p>
<h2 id="scalable-high-performance-indexing">Scalable, High-Performance Indexing</h2>
<ul>
<li>over <a href="http://home.apache.org/~mikemccand/lucenebench/indexing.html">150GB/hour on modern hardware</a></li>
<li>small RAM requirements -- only 1MB heap</li>
<li>incremental indexing as fast as batch indexing</li>
<li>index size roughly 20-30% the size of text indexed</li>
</ul>
""".trimIndent()
index("first", content = content)
assertThat(search(content = "fast"))
.hasSize(1)
.anyMatch { it.title == "first" }
@Suppress("SpellCheckingInspection")
assertThat(search(content = "preformance")) // <- note the error
.hasSize(1)
.anyMatch { it.title == "first" }
}
@Test
fun `combined search`() {
@Language("html")
val content =
"""
<div>
<h1 class="title">Apache Lucene Core</h1>
<p>Apache Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> is a
high-performance, full-featured text search engine library written entirely in Java.
It is a technology suitable for nearly any application that requires full-text search,
especially cross-platform.</p>
<p>Apache Lucene is an open source project available for free download. Please use the
links on the right to access Lucene.</p>
<h1 id="lucenetm-features">Lucene<span style="vertical-align: super; font-size: xx-small">TM</span> Features</h1>
<p>Lucene offers powerful features through a simple API:</p>
<h2 id="scalable-high-performance-indexing">Scalable, High-Performance Indexing</h2>
<ul>
<li>over <a href="http://home.apache.org/~mikemccand/lucenebench/indexing.html">150GB/hour on modern hardware</a></li>
<li>small RAM requirements -- only 1MB heap</li>
<li>incremental indexing as fast as batch indexing</li>
<li>index size roughly 20-30% the size of text indexed</li>
</ul>
""".trimIndent()
index("first", content = content, tags = listOf("abc"))
assertThat(search(title = "fir", tag = "abc", content = "20"))
.hasSize(1)
}
@Test
fun `delete index`() {
val uuid = index("first").uuid
searcher.deleteIndex(1, uuid)
assertThat(search("first")).isEmpty()
}
@Test
fun `update index`() {
val note = index("first")
searcher.updateIndex(1, note.copy(meta = note.meta.copy(title = "new")))
assertThat(search("first")).isEmpty()
assertThat(search("new")).hasSize(1)
}
}