diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..73f69e0 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..7d531f9 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +computer-vision \ No newline at end of file diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 0000000..1a7cd8e --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,88 @@ + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..a55e7a1 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..b589d56 --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/gradle.xml b/.idea/gradle.xml new file mode 100644 index 0000000..9d1a244 --- /dev/null +++ b/.idea/gradle.xml @@ -0,0 +1,18 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/jarRepositories.xml b/.idea/jarRepositories.xml new file mode 100644 index 0000000..fdc392f --- /dev/null +++ b/.idea/jarRepositories.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries-with-intellij-classes.xml b/.idea/libraries-with-intellij-classes.xml new file mode 100644 index 0000000..9fa3156 --- /dev/null +++ b/.idea/libraries-with-intellij-classes.xml @@ -0,0 +1,65 @@ + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..fe0b0da --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml new file mode 100644 index 0000000..e96534f --- /dev/null +++ b/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 0000000..df449d7 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,35 @@ +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + kotlin("jvm") version "1.6.10" + application +} + +group = "network.rs485.ben" +version = "1.0-SNAPSHOT" + +repositories { + mavenCentral() +} + +dependencies { + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.0") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-swing:1.6.0") + implementation("org.jetbrains.kotlin:kotlin-annotations-jvm:1.6.10") + implementation("io.ktor:ktor-io:1.6.7") + implementation(files("/usr/share/java/opencv.jar")) // provided by the OS (Arch Linux) after installation + testImplementation(kotlin("test")) +} + +tasks.test { + useJUnitPlatform() +} + +tasks.withType() { + kotlinOptions.jvmTarget = "17" + kotlinOptions.freeCompilerArgs += "-Xopt-in=kotlin.RequiresOptIn" +} + +application { + mainClass.set("network.rs485.ben.computervision.MainKt") +} diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..7fc6f1f --- /dev/null +++ b/gradle.properties @@ -0,0 +1 @@ +kotlin.code.style=official diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..e708b1c Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..84d1f85 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..4f906e0 --- /dev/null +++ b/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..107acd3 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 0000000..79c73cc --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,3 @@ + +rootProject.name = "computer-vision" + diff --git a/src/main/kotlin/network/rs485/ben/computervision/Color.kt b/src/main/kotlin/network/rs485/ben/computervision/Color.kt new file mode 100644 index 0000000..2fea9a1 --- /dev/null +++ b/src/main/kotlin/network/rs485/ben/computervision/Color.kt @@ -0,0 +1,37 @@ +package network.rs485.ben.computervision + +import network.rs485.ben.computervision.CvContext.withMat +import org.opencv.core.CvType +import org.opencv.core.Mat +import org.opencv.core.Scalar +import org.opencv.imgproc.Imgproc + +internal val COLOR_BLACK = Scalar(0.0, 0.0, 0.0, 255.0) +internal val COLOR_WHITE = Scalar(255.0, 255.0, 255.0, 255.0) +internal val COLOR_RED = Scalar(0.0, 0.0, 180.0, 255.0) +internal val COLOR_GREEN = Scalar(0.0, 255.0, 0.0, 255.0) +internal val COLOR_BLUE = Scalar(255.0, 0.0, 0.0, 255.0) +internal val COLOR_YELLOW = Scalar(0.0, 240.0, 255.0, 255.0) + +internal operator fun Scalar.plus(other: Scalar): Scalar = + Scalar((0..3).map { `val`[it] + other.`val`[it] }.toDoubleArray()) + +internal operator fun Scalar.minus(other: Scalar): Scalar = + Scalar((0..3).map { `val`[it] - other.`val`[it] }.toDoubleArray()) + +internal operator fun Scalar.times(factor: Double): Scalar = + Scalar((0..3).map { `val`[it] * factor }.toDoubleArray()) + +internal operator fun Scalar.div(divider: Double): Scalar = + Scalar((0..3).map { (`val`[it] / divider) }.toDoubleArray()) + +internal fun Scalar.asArray(length: Int = 4): DoubleArray { + assert(length > 0) + return (0 until length).map { `val`.getOrElse(it) { 0.0 } }.toDoubleArray() +} + +internal fun Mat.copyToColor(image: Mat) = withMat(MatInfo.from(image, CvType.CV_8U)) { tmp -> + convertTo(tmp, CvType.CV_8U) + Imgproc.cvtColor(tmp, image, Imgproc.COLOR_GRAY2BGR) + image +} diff --git a/src/main/kotlin/network/rs485/ben/computervision/CvContext.kt b/src/main/kotlin/network/rs485/ben/computervision/CvContext.kt new file mode 100644 index 0000000..5993ef9 --- /dev/null +++ b/src/main/kotlin/network/rs485/ben/computervision/CvContext.kt @@ -0,0 +1,110 @@ +package network.rs485.ben.computervision + +import io.ktor.utils.io.pool.* +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.opencv.core.* +import org.opencv.imgcodecs.Imgcodecs +import org.opencv.imgproc.Imgproc +import java.nio.file.Path + +data class MatInfo(val rows: Int, val cols: Int, val type: Int) { + companion object { + fun from(mat: Mat) = MatInfo(mat.rows(), mat.cols(), mat.type()) + fun from(mat: Mat, type: Int) = MatInfo(mat.rows(), mat.cols(), type) + } + fun create(): Mat = Mat(rows, cols, type) +} + +private val histMatInfo = MatInfo(256, 1, CvType.CV_32FC1) +private val predefinedChannels = listOf(16, 24, 64, 128) +private val matPoolsToCreate = listOf( + MatInfo(32, 32, CvType.CV_8UC1) to Runtime.getRuntime().availableProcessors() * 4, + histMatInfo to Runtime.getRuntime().availableProcessors() * 4, +) + predefinedChannels.map { + MatInfo(it, it, CvType.CV_8UC1) to Runtime.getRuntime().availableProcessors() * 2 +} + +object CvContext { + lateinit var gaussianKernel: Mat + lateinit var histChannelPar: MatOfInt + lateinit var histSizePar: MatOfInt + lateinit var histRangesPar: MatOfFloat + lateinit var matPools: HashMap + + fun initialize() { + System.loadLibrary(Core.NATIVE_LIBRARY_NAME) + this.gaussianKernel = Imgproc.getGaussianKernel(GAUSSIAN_KERNEL_SIZE, -1.0) + this.histChannelPar = MatOfInt(0) + this.histSizePar = MatOfInt(HIST_SIZE) + this.histRangesPar = MatOfFloat(0f, 255f) + this.matPools = matPoolsToCreate.associateTo(HashMap()) { (matInfo, capacity) -> + matInfo to MatPool(matInfo, capacity) + } + } + + internal inline fun withGrayCalcHist(image: Mat, mask: Mat, block: (Mat) -> T): T = withMat(histMatInfo) { hist -> + Imgproc.calcHist( + /* images = */ listOf(image), + /* channels = */ histChannelPar, + /* mask = */ mask, + /* hist = */ hist, + /* histSize = */ histSizePar, + /* ranges = */ histRangesPar, + ) + block(hist) + } + + internal inline fun withRotationMatrix(angle: Double, center: Point, block: (Mat) -> T): T = + Imgproc.getRotationMatrix2D( + /* center = */ center, + /* angle = */ Math.toDegrees(angle), + /* scale = */ 1.0, + ).letAutoclean(block) + + class MatPool( + private val info: MatInfo, + capacity: Int, + ) : DefaultPool(capacity) { + override fun produceInstance(): Mat = info.create() + } + + internal inline fun withMat(matInfo: MatInfo, block: (Mat) -> T): T { + val upperMatInfo = if (matInfo.type == CvType.CV_8UC1) { + predefinedChannels.find { matInfo.rows <= it && matInfo.cols <= it } + ?.let { MatInfo(it, it, CvType.CV_8UC1) } + ?: matInfo + } else matInfo + return if (matPools.containsKey(upperMatInfo)) { + val pool = matPools[upperMatInfo]!! + val mat = pool.borrow() + autoclean(mat, { pool.recycle(mat) }, block) + } else { + println("Not pooled: $matInfo") + matInfo.create().letAutoclean(block) + } + } + + internal fun recycle(image: Mat) = matPools[MatInfo.from(image)]?.recycle(image) ?: image.release() + + internal suspend fun readAndConvertImage(imagePath: Path): Pair { + val displayImage = withContext(Dispatchers.IO) { + Imgcodecs.imread(imagePath.toString()) + } ?: throw RuntimeException("Could not load image from $imagePath") + println("Loaded image $imagePath as $displayImage") + val matInfo = MatInfo(displayImage.rows(), displayImage.cols(), CvType.CV_8UC1) + val image = matPools.computeIfAbsent(matInfo) { + MatPool(matInfo, Runtime.getRuntime().availableProcessors()) + }.borrow() + Imgproc.cvtColor(displayImage, image, Imgproc.COLOR_BGR2GRAY) + return displayImage to image + } +} + +internal inline fun autoclean(obj: V, release: () -> Unit, block: (V) -> T): T = + runCatching { block(obj) }.also { release() }.getOrThrow() + +internal inline fun Mat.letAutoclean(block: (Mat) -> T): T = + autoclean(this, this::release, block) + +internal inline fun withTmpMat(block: (tmp: Mat) -> T): T = Mat().letAutoclean(block) diff --git a/src/main/kotlin/network/rs485/ben/computervision/DimensionProvider.kt b/src/main/kotlin/network/rs485/ben/computervision/DimensionProvider.kt new file mode 100644 index 0000000..34fa00e --- /dev/null +++ b/src/main/kotlin/network/rs485/ben/computervision/DimensionProvider.kt @@ -0,0 +1,24 @@ +package network.rs485.ben.computervision + +import org.opencv.core.Mat + +interface DimensionProvider { + val minX: Int + val maxX: Int + val minY: Int + val maxY: Int +} + +class CroppingDimensionProvider(lines: List) : DimensionProvider { + override val minX: Int = lines.minOf { it.first.x } + override val maxX: Int = lines.maxOf { it.second.x } + override val minY: Int = lines.first().first.y + override val maxY: Int = lines.last().first.y +} + +class ImageDimensionProvider(image: Mat) : DimensionProvider { + override val minX: Int = 0 + override val maxX: Int = image.cols() - 1 + override val minY: Int = 0 + override val maxY: Int = image.rows() - 1 +} diff --git a/src/main/kotlin/network/rs485/ben/computervision/Entity.kt b/src/main/kotlin/network/rs485/ben/computervision/Entity.kt new file mode 100644 index 0000000..2010690 --- /dev/null +++ b/src/main/kotlin/network/rs485/ben/computervision/Entity.kt @@ -0,0 +1,73 @@ +package network.rs485.ben.computervision + +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import org.opencv.core.Mat +import org.opencv.core.Scalar +import java.util.* + +class Entity { + val lines: TreeSet = sortedSetOf(comparator = LineComparator) + + fun color(color: Scalar, image: Mat) { + lines.forEach { + image.drawLine(it, color) + } + } + + fun merge(other: Entity) { + lines.addAll(other.lines) + } + + override fun equals(other: Any?): Boolean { + return this === other || other is Entity && lines == other.lines + } + + override fun hashCode(): Int { + return lines.hashCode() + } + + override fun toString(): String { + return "Entity(${lines.joinToString(limit = 3) { "${it.first} -- ${it.second}" }})" + } +} + +fun MutableList.merge(entity: Entity) { + val toMerge = mutableSetOf() + entity.lines.forEach { line -> + toMerge.addAll(this.filter { otherEntity -> + otherEntity.lines.any(line::intersects) + }) + } + if (toMerge.size == 0) { + add(entity) + } else { + val retainedEntity = toMerge.first() + retainedEntity.merge(entity) + toMerge.forEach { + if (it !== retainedEntity) { + this.remove(it) + retainedEntity.merge(it) + } + } + } +} + +suspend fun MutableList.colorRanged( + start: Scalar, + end: Scalar, + image: Mat, + windowMgr: IWindowManager, +) = coroutineScope { + if (size == 0) return@coroutineScope + if (size == 1) get(0).color(start, image) + val direction = (end - start) * (1.0 / (size - 1)) + forEachIndexed { index, entity -> + entity.color(start + (direction * index.toDouble()), image) + launch { + windowMgr.updateImage(image) + } + if (ANIMATION_DELAY.isPositive()) delay(ANIMATION_DELAY) + } +} diff --git a/src/main/kotlin/network/rs485/ben/computervision/EntityScanner.kt b/src/main/kotlin/network/rs485/ben/computervision/EntityScanner.kt new file mode 100644 index 0000000..35bc22b --- /dev/null +++ b/src/main/kotlin/network/rs485/ben/computervision/EntityScanner.kt @@ -0,0 +1,194 @@ +package network.rs485.ben.computervision + +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.* +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.opencv.core.Mat + +class EntityScanner(private val animate: Boolean, private val scanBarSpacing: Int, private val foregroundThreshold: Float) { + + private fun Mat.isValid(row: Int, column: Int): Boolean = + (this[row, column]?.getOrNull(0) ?: 255.0) < foregroundThreshold + + private fun Mat.isValid(point: Pixel): Boolean = + (this[point]?.getOrNull(0) ?: 255.0) < foregroundThreshold + + suspend fun scanImage(image: Mat, displayImage: Mat, windowMgr: IWindowManager) = coroutineScope { + val entities = mutableListOf() + val mergeMutex = Mutex() + lineScan(image, displayImage, windowMgr) + .map { line -> + lineGrowToEntityDown(line, image, displayImage, windowMgr).also { entity -> + // add missing upper parts of all entities + lineGrowToEntityUp(line, image, displayImage, windowMgr, entity) + } + } + .collect { + mergeMutex.withLock { + entities.merge(it) + } + } + entities.removeIf { it.lines.size < 6 } + if (animate) { + entities.colorRanged(COLOR_GREEN, COLOR_RED, displayImage, windowMgr) + windowMgr.updateImage(displayImage) + } + return@coroutineScope entities + } + + @OptIn(ExperimentalCoroutinesApi::class) + private suspend fun lineScan( + image: Mat, + displayImage: Mat, + windowMgr: IWindowManager, + ) = channelFlow { + coroutineScope { + for (i in 0 until image.height().div(scanBarSpacing)) { + launch { + val lineNumber = i * scanBarSpacing + var start = Pixel(0, lineNumber) + var end = Pixel(image.width() - 1, lineNumber) + if (animate) { + displayImage.drawLine(Line(start, end), COLOR_RED) + launch { + windowMgr.updateImage(displayImage) + } + } + var onEntity = false + for (x in 0 until image.width()) { + val color = if (image.isValid(lineNumber, x)) { + COLOR_GREEN.also { + if (!onEntity) { + onEntity = true + start = Pixel(x, start.y) + } + } + } else { + COLOR_BLUE.also { + if (onEntity) { + onEntity = false + send(start to Pixel(x, start.y)) + } + } + } + end = Pixel(x, end.y) + if (animate) { + // color the current dot on the line + displayImage.put(lineNumber, x, *color.asArray(3)) + launch { + windowMgr.updateImage(displayImage) + } + if (ANIMATION_DELAY.isPositive()) delay(ANIMATION_DELAY) + } + } + if (onEntity) { + send(start to Pixel(image.width() - 1, start.y)) + } + } + } + } + } + + private suspend fun lineGrowToEntityDown( + line: Line, + image: Mat, + displayImage: Mat, + windowMgr: IWindowManager, + entity: Entity = Entity(), + ) = lineGrowToEntity( + line = line, + image = image, + displayImage = displayImage, + windowMgr = windowMgr, + entity = entity, + last = (line.first.y + scanBarSpacing - 1).coerceIn(0, image.height() - 1), + step = 1, + ) + + private suspend fun lineGrowToEntityUp( + line: Line, + image: Mat, + displayImage: Mat, + windowMgr: IWindowManager, + entity: Entity = Entity(), + ) = lineGrowToEntity( + line = line, + image = image, + displayImage = displayImage, + windowMgr = windowMgr, + entity = entity, + last = (line.first.y - scanBarSpacing + 1).coerceIn(0, image.height() - 1), + step = -1, + ) + + private suspend fun lineGrowToEntity( + line: Line, + image: Mat, + displayImage: Mat, + windowMgr: IWindowManager, + entity: Entity, + last: Int, + step: Int, + ): Entity = coroutineScope { + assert(step != 0) + + suspend fun lineFound(foundLine: Line) { + assert(foundLine.first.x >= 0) + assert(foundLine.first.y >= 0) + assert(foundLine.first.x <= foundLine.second.x) + assert(foundLine.first.y == foundLine.second.y) + entity.lines.add(foundLine) + if (animate) { + displayImage.drawLine(foundLine, COLOR_YELLOW) + launch { + windowMgr.updateImage(displayImage) + } + if (ANIMATION_DELAY.isPositive()) delay(ANIMATION_DELAY) + } + } + + lineFound(line) + + suspend fun lineGrowRecursion(foundLine: Line) { + lineFound(foundLine) + if (step > 0 && foundLine.first.y <= last || step < 0 && foundLine.first.y >= last) { + lineGrowToEntity( + line = foundLine, + image = image, + displayImage = displayImage, + windowMgr = windowMgr, + entity = entity, + last = last, + step = step, + ) + } + } + + var start = Pixel(line.first.x, line.first.y + step) + if (start.y < 0 || start.y >= image.height()) return@coroutineScope entity + val row = start.y + fun IntProgression.lastValid() = takeWhile { image.isValid(row, it) }.last() + var end = start + if (image.isValid(start)) { + start = Pixel(line.first.x.downTo(0).lastValid(), start.y) + end = Pixel(line.first.x.until(image.width()).lastValid(), end.y) + lineGrowRecursion(start to end) + } + var next = end.x + 2 + while (next <= line.second.x) { + if (image.isValid(row, next)) { + start = Pixel(end.x + 2, start.y) + end = Pixel(next.until(image.width()).lastValid(), start.y) + lineGrowRecursion(start to end) + next = end.x + 2 + } else { + ++next + } + } + return@coroutineScope entity + } +} diff --git a/src/main/kotlin/network/rs485/ben/computervision/Golden.kt b/src/main/kotlin/network/rs485/ben/computervision/Golden.kt new file mode 100644 index 0000000..af54a44 --- /dev/null +++ b/src/main/kotlin/network/rs485/ben/computervision/Golden.kt @@ -0,0 +1,120 @@ +package network.rs485.ben.computervision + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch +import org.opencv.core.* +import org.opencv.imgproc.Imgproc +import java.nio.file.Path +import kotlin.math.roundToInt + +internal const val DISTANCE_DEVIATION: Double = 1.0 +internal const val DISTANCE_STEP = 0.5 +internal const val DILATION_SIZE = 1.0 + +fun interface DeviationFunction { + fun calculate(level: Int, value: Double): Double +} + +data class DeviationParameters(val deviationMin: Int, val deviationFunc: DeviationFunction, val maximumDeviation: Double) { + val deviationRange = deviationMin until HIST_SIZE +} + +class Golden( + val goldenName: String, + val goldenColor: Scalar, + entity: Entity, + val image: Mat, + val displayImage: Mat, + val windowManager: IWindowManager, + val deviationParameters: DeviationParameters, +) : IdentifiableEntity(entity, ImageDimensionProvider(image)) { + companion object { + suspend fun createGolden( + parentScope: CoroutineScope, + scanner: EntityScanner, + goldenName: String, + goldenPath: Path, + goldenColor: Scalar, + deviationParameters: DeviationParameters, + ): Golden { + val windowMgr = WINDOW_MANAGER_FACTORY.createWindowManager(goldenName) + val (displayImage, image) = CvContext.readAndConvertImage(goldenPath) + parentScope.launch { + windowMgr.display(displayImage) + } + windowMgr.setDimension(256, 256) + val entity = + scanner.scanImage(image, displayImage, windowMgr) + .takeIf { it.size == 1 } + ?.get(0) + ?: throw RuntimeException("Cannot find $goldenName in golden image") + val golden = Golden( + goldenName = goldenName, + goldenColor = goldenColor, + entity = entity, + image = image, + displayImage = displayImage, + windowManager = windowMgr, + deviationParameters = deviationParameters, + ) + golden.distanceMat.copyToColor(displayImage) + parentScope.launch { + golden.updateImage() + } + return golden + } + } + + val orientation: Line = run { + val midX = minMaxDistance.maxLoc.x.roundToInt() + val midY = minMaxDistance.maxLoc.y.roundToInt() + val midRounded = Pixel(midX, midY) + if (minMaxDistance.maxLoc != midRounded.vec2D()) println("${minMaxDistance.maxLoc} != ${midRounded.vec2D()}") + transformedLines.filter { it.first.y >= midY } + .flatMap { (first, second) -> listOf(first, second) } + .map { startPixel -> + val endPixel = entityMask.searchGreatest( + center = midRounded, + direction = (midRounded - startPixel).vec2D().normalize(), + searchRange = (0..(cols + rows)) // use distance step? + ) ?: throw RuntimeException( + "Could not find any border for $goldenName center $midRounded in $entityMask:\n" + + entityMask.dump() + ) + startPixel to endPixel + } + .maxByOrNull { it.vec2D().magnitude } + ?: throw IllegalStateException("Could not find orientation for $goldenName") + } + + val orientationRadius = orientation.vec2D().magnitude / 2.0 + + val entityMaskDilated: Mat = Mat(rows, cols, entityMask.type()).also { dest -> + val dilationKernel = Imgproc.getStructuringElement( + /* shape = */ Imgproc.MORPH_ELLIPSE, + /* ksize = */ Size(2 * DILATION_SIZE + 1, 2 * DILATION_SIZE + 1), + ) + Imgproc.dilate(entityMask, dest, dilationKernel) + } + + internal inline fun withCentrumMask(identity: IdentifiableEntity, block: (Mat) -> T): T = + CvContext.withMat(MatInfo(identity.rows, identity.cols, CvType.CV_8UC1)) { maskMat -> + Core.inRange( + /* src = */ identity.distanceMat, + /* lowerb = */ Scalar(minMaxDistance.maxVal.minus(DISTANCE_DEVIATION)), + /* upperb = */ Scalar(minMaxDistance.maxVal.plus(DISTANCE_DEVIATION)), + /* dst = */ maskMat, + ) + block(maskMat) + } + + suspend fun updateImage() = windowManager.updateImage(displayImage) + + override fun release() { + super.release() + entityMaskDilated.release() + windowManager.dispose() + } + + override fun toString(): String = "Golden($goldenName)" +} diff --git a/src/main/kotlin/network/rs485/ben/computervision/IdentifiableEntity.kt b/src/main/kotlin/network/rs485/ben/computervision/IdentifiableEntity.kt new file mode 100644 index 0000000..c294e8e --- /dev/null +++ b/src/main/kotlin/network/rs485/ben/computervision/IdentifiableEntity.kt @@ -0,0 +1,55 @@ +package network.rs485.ben.computervision + +import org.opencv.core.* +import org.opencv.imgproc.Imgproc + +open class IdentifiableEntity( + entity: Entity, + private val dimensionProvider: DimensionProvider = CroppingDimensionProvider(entity.lines.toList()), +) { + companion object { + const val DIST_TRANSFORM_MASK_SIZE: Int = 3 + + // DIST_L2 is the simple euclidean distances + const val DIST_TRANSFORM_FUNC = Imgproc.DIST_L2 + } + + val rows: Int = dimensionProvider.maxY - dimensionProvider.minY + 1 + val cols: Int = dimensionProvider.maxX - dimensionProvider.minX + 1 + val rect: Rect + get() = Rect(dimensionProvider.minX, dimensionProvider.minY, cols, rows) + val lines = entity.lines.toList() + val transformedLines = lines.map { + transformPixel(it.first) to transformPixel(it.second) + } + val entityMask: Mat = Mat(rows, cols, CvType.CV_8UC1).also { maskMat -> + maskMat.setTo(COLOR_BLACK) + transformedColor(COLOR_WHITE, maskMat) + } + val distanceMat: Mat = Mat(rows, cols, CvType.CV_32FC1).also { distanceMat -> + Imgproc.distanceTransform(entityMask, distanceMat, DIST_TRANSFORM_FUNC, DIST_TRANSFORM_MASK_SIZE) + } + val minMaxDistance: Core.MinMaxLocResult by lazy { Core.minMaxLoc(distanceMat, entityMask) } + + fun transformedColor(color: Scalar, image: Mat) { + transformedLines.forEach { + image.drawLine(it, color) + } + } + + fun transformPixel(pixel: Pixel) = Pixel(pixel.x - dimensionProvider.minX, pixel.y - dimensionProvider.minY) + + open fun release() { + entityMask.release() + distanceMat.release() + } + + override fun toString(): String { + return "IdentifiableEntity($rect)" + } +} + +internal inline fun withIdEntity(entity: Entity, block: (identity: IdentifiableEntity) -> T): T = + IdentifiableEntity(entity).let { identity -> + autoclean(identity, identity::release, block) + } diff --git a/src/main/kotlin/network/rs485/ben/computervision/Math2D.kt b/src/main/kotlin/network/rs485/ben/computervision/Math2D.kt new file mode 100644 index 0000000..90e15f2 --- /dev/null +++ b/src/main/kotlin/network/rs485/ben/computervision/Math2D.kt @@ -0,0 +1,111 @@ +package network.rs485.ben.computervision + +import org.opencv.core.* +import org.opencv.imgproc.Imgproc +import kotlin.math.pow +import kotlin.math.roundToInt +import kotlin.math.sqrt + +internal typealias Pixel = Pair +internal typealias Line = Pair + +internal val Pixel.x: Int + get() = first + +internal val Pixel.y: Int + get() = second + +internal operator fun Pixel.plus(other: Pixel): Pixel = Pixel(x + other.x, y + other.y) + +internal operator fun Pixel.minus(other: Pixel): Pixel = Pixel(x - other.x, y - other.y) + +internal operator fun Mat.get(pixel: Pixel): DoubleArray? = + get(/* row = */ pixel.y, /* col = */ pixel.x) + +fun Point.rounded(): Pixel = x.roundToInt() to y.roundToInt() + +internal operator fun Mat.get(point: Point): DoubleArray? = + get(/* row = */ point.y.toInt(), /* col = */ point.x.toInt()) + +class Vector2D(x: Double, y: Double) : Point(x, y) { + val magnitude: Double + get() = sqrt(x.pow(2.0) + y.pow(2.0)) + + constructor(other: Point) : this(other.x, other.y) + + fun normalize() = apply { + magnitude.let { + x /= it + y /= it + } + } + + operator fun times(factor: Double): Vector2D = Vector2D(x * factor, y * factor) + + operator fun div(divider: Double): Vector2D = Vector2D(x / divider, y / divider) + + operator fun plus(point: Point) = Vector2D(x + point.x, y + point.y) + +} + +@JvmName("pixelToVec2D") +internal fun Pixel.vec2D(): Vector2D = Vector2D(x.toDouble(), y.toDouble()) + +@JvmName("lineToVec2D") +internal fun Line.vec2D(): Vector2D = Vector2D((second.x - first.x).toDouble(), (second.y - first.y).toDouble()) + +internal object LineComparator : Comparator { + override fun compare(line1: Line, line2: Line): Int { + assert(line1.first.y == line1.second.y) + assert(line2.first.y == line2.second.y) + val rowCompare = line1.first.y.compareTo(line2.first.y) + if (rowCompare != 0) return rowCompare + val lineStartCompare = line1.first.x.compareTo(line2.first.x) + if (lineStartCompare != 0) return lineStartCompare + return line1.second.x.compareTo(line2.second.x) + } +} + +internal fun Line.intersects(other: Line): Boolean { + assert(first.y == second.y) + assert(other.first.y == other.second.y) + if (first.y != other.first.y) return false + if (first.x == other.first.x) return true + return if (first.x < other.first.x) { + second.x >= other.first.x + } else { + other.second.x >= first.x + } +} + +internal fun Mat.drawLine(line: Line, color: Scalar) = Imgproc.line( + /* img = */ this, + /* pt1 = */ line.first.vec2D(), + /* pt2 = */ line.second.vec2D(), + /* color = */ color, +) + +internal fun Mat.filterPixels(filterFunc: (value: DoubleArray) -> Boolean): List = + (0 until rows()).flatMap { row -> + (0 until cols()) + .filter { col -> filterFunc(get(row, col)) } + .map { col -> Pixel(col, row) } + } + +internal fun Mat.sumOfHist( + indices: IntRange, + maximum: Double, + mapFunc: (level: Int, value: Double) -> Double, +): Double { + var sum: Double = 0.toDouble() + for (idx in indices) { + sum += mapFunc(idx, get(idx, 0)[0]) + if (sum > maximum) break + } + return sum +} + +fun Mat.searchGreatest(center: Pixel, direction: Vector2D, searchRange: Iterable): Pixel? = + searchRange.map { factor: Number -> (direction * factor.toDouble()).rounded() + center } + .filter { it.x in (0 until width()) && it.y in (0 until height()) } + .lastOrNull { pixel -> get(pixel)?.let { it[0] > 0 } ?: false } diff --git a/src/main/kotlin/network/rs485/ben/computervision/WindowManager.kt b/src/main/kotlin/network/rs485/ben/computervision/WindowManager.kt new file mode 100644 index 0000000..6595611 --- /dev/null +++ b/src/main/kotlin/network/rs485/ben/computervision/WindowManager.kt @@ -0,0 +1,129 @@ +package network.rs485.ben.computervision + +import kotlinx.coroutines.* +import org.opencv.core.Mat +import org.opencv.highgui.HighGui +import org.opencv.highgui.ImageWindow +import java.awt.event.KeyEvent +import java.awt.event.KeyListener +import java.awt.event.WindowAdapter +import java.awt.event.WindowEvent +import java.awt.image.BufferedImage +import java.util.concurrent.atomic.AtomicInteger +import javax.swing.ImageIcon +import javax.swing.JLabel +import kotlin.system.exitProcess + +val DEFAULT_WINDOW_MANAGER_FACTORY = WindowManagerFactory(::WindowManager) +val NOOP_WINDOW_MANAGER_FACTORY = WindowManagerFactory { NoopWindowManager } + +fun interface WindowManagerFactory { + fun createWindowManager(windowName: String): IWindowManager +} + +interface IWindowManager { + suspend fun display(img: Mat) + suspend fun updateImage(img: Mat) + fun dispose() + fun setDimension(width: Int, height: Int) +} + +private object NoopWindowManager : IWindowManager { + override suspend fun display(img: Mat) = Unit + override suspend fun updateImage(img: Mat) = Unit + override fun dispose() = Unit + override fun setDimension(width: Int, height: Int) = Unit +} + +private class WindowManager(windowName: String) : IWindowManager { + private var updateState: AtomicInteger = AtomicInteger(2) + private var disposeDeferred: CompletableDeferred? = null + private val window: ImageWindow = ImageWindow(windowName, 0) + + override suspend fun display(img: Mat) = withContext(Dispatchers.Main) { + if (disposeDeferred != null) { + throw IllegalStateException("A window is already shown") + } + window.setMat(img) + window.show() + updateState.set(0) + val deferred = CompletableDeferred() + disposeDeferred = deferred + try { + window.frame.addWindowListener(object : WindowAdapter() { + override fun windowClosed(e: WindowEvent?) { + deferred.cancel("Window closed") + } + }) + window.frame.addKeyListener(object : KeyListener { + override fun keyTyped(e: KeyEvent) {} + + override fun keyPressed(e: KeyEvent) {} + + override fun keyReleased(e: KeyEvent) { + if (e.keyCode == KeyEvent.VK_ESCAPE) { + deferred.cancel("Escape pressed") + } + } + }) + deferred.await() + } finally { + disposeDeferred = null + window.dispose() + } + } + + override suspend fun updateImage(img: Mat) { + window.setMat(img) + val state = updateState.getAndUpdate { it.plus(1).coerceAtMost(3) } + if (state == 3) return + withContext(Dispatchers.Main) { + updateState.updateAndGet { it.minus(1).coerceAtLeast(0) } + window.show() + } + } + + override fun dispose() { + disposeDeferred?.complete(Unit) + } + + override fun setDimension(width: Int, height: Int) = window.setNewDimension(width, height) + + /** + * @see org.opencv.highgui.HighGui.waitKey(Int) + */ + private fun ImageWindow.show() { + if (img != null) { + val bufferedImage = try { + HighGui.toBufferedImage(img) + } catch (error: UnsupportedOperationException) { + System.err.println("Cannot show image ($img) in window ${window.name}: ${error.message}") + BufferedImage(img.width(), img.height(), BufferedImage.TYPE_INT_RGB).also { + for (y in 0 until it.height) { + if (y % 2 == 1) { + for (x in 0 until it.width) { + it.setRGB(x, y, 0xff00ff) + } + } + } + } + } + val imageIcon = ImageIcon(bufferedImage) + if (lbl == null) { + val frame = HighGui.createJFrame(name, flag) + val lbl = JLabel(imageIcon) + setFrameLabelVisible(frame, lbl) + } else { + lbl.icon = imageIcon + } + } else { + System.err.println("Error: no imshow associated with namedWindow: \"$name\"") + exitProcess(-1) + } + } + + private fun ImageWindow.dispose() { + frame.dispose() + } + +} diff --git a/src/main/kotlin/network/rs485/ben/computervision/main.kt b/src/main/kotlin/network/rs485/ben/computervision/main.kt new file mode 100644 index 0000000..c2be0eb --- /dev/null +++ b/src/main/kotlin/network/rs485/ben/computervision/main.kt @@ -0,0 +1,481 @@ +@file:OptIn(FlowPreview::class, ExperimentalCoroutinesApi::class) + +package network.rs485.ben.computervision + +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.ProducerScope +import kotlinx.coroutines.flow.* +import org.opencv.core.* +import org.opencv.imgproc.Imgproc +import java.io.FileNotFoundException +import java.nio.file.FileSystemNotFoundException +import java.nio.file.Path +import java.nio.file.Paths +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import kotlin.io.path.* +import kotlin.math.acos +import kotlin.system.exitProcess +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds + +internal const val HIST_SIZE = 256 +internal const val FOREGROUND_THRESHOLD = 120.0f +internal const val GAUSSIAN_KERNEL_SIZE = 3 +internal val COROUTINE_BUFFER_SIZE = System.getProperties().getProperty("cv.coroutines.buffer.size", "4").toInt() +internal val MAXIMUM_PARALLEL_IMAGES = System.getProperties().getProperty("cv.images.buffer.size", "1").toInt() +//internal val ANIMATION_DELAY: Duration = 100.milliseconds +internal val ANIMATION_DELAY: Duration = Duration.ZERO +//internal val WINDOW_MANAGER_FACTORY = DEFAULT_WINDOW_MANAGER_FACTORY +internal val WINDOW_MANAGER_FACTORY = NOOP_WINDOW_MANAGER_FACTORY +private val SINGLE_DISPATCHER = Executors.newSingleThreadExecutor().asCoroutineDispatcher() + +internal object Resource { + fun getPath(name: String): Path { + val path = try { + javaClass.getResource("/$name")?.toURI()?.let(Paths::get) + } catch (error: FileSystemNotFoundException) { + System.err.println("Could not read resource $name, error:\n $error\n using fallback..") + null + } ?: Paths.get(name).apply { + if (!exists()) { + throw FileNotFoundException("Cannot find $this on filesystem $fileSystem") + } + } + return path + } +} + +typealias Candidate = Triple + +private suspend fun findGoldenPoI( + image: Mat, + identity: IdentifiableEntity, + golden: Golden, + centrumMask: Mat, +): List>> { + val goldenOrientation = golden.orientation.vec2D() + val goldenDirection = Vector2D(goldenOrientation).normalize() + val searchMag = golden.orientationRadius + .let { (it - DISTANCE_DEVIATION..it + DISTANCE_DEVIATION) } + val bestCenters = channelFlow bestCenters@{ + for (center in centrumMask.filterPixels { it[0] > 0.0 }) { + launch findCenterDeviation@{ + val intoGoldSpace = golden.minMaxDistance.maxLoc.rounded() - center + val rect = golden.rect.apply { + x += identity.rect.x - intoGoldSpace.x + y += identity.rect.y - intoGoldSpace.y + } + if ( + rect.x < 0 || + rect.y < 0 || + rect.x + rect.width >= image.width() || + rect.y + rect.height >= image.height() + ) { + if (ANIMATION_DELAY.isPositive()) println("Not considering $center for $identity on $golden") + return@findCenterDeviation + } + image.withSubmatSafe(rect) { subImage -> + val deviationFlow = channelFlow { + suspend fun ProducerScope.emitDeviation(startPixel: Pixel) { + val centerStart = center - startPixel + if (centerStart == Pixel(0, 0)) return + val dirVec = centerStart.vec2D().normalize() + val endpix = try { + identity.entityMask.searchGreatest( + center = center, + direction = dirVec, + searchRange = searchMag.progression(step = DISTANCE_STEP) + ) + } catch (error: IllegalArgumentException) { + System.err.println( + "Error when finding greatest magnitude of $center for $identity.\n" + + " Values: center = $center, dir = $dirVec, searchMag = $searchMag\n" + + " $error" + ) + null + } ?: return + val angle = acos(dirVec.dot(goldenDirection)) + val deviation = CvContext.withMat(MatInfo.from(subImage)) { tmp -> + subImage.copyTo(tmp) + CvContext.withRotationMatrix( + angle = angle, + center = golden.minMaxDistance.maxLoc, + ) { intoGoldSpaceRot -> + Imgproc.warpAffine(tmp, tmp, intoGoldSpaceRot, tmp.size()) + } + Core.absdiff(golden.image, tmp, tmp) + Imgproc.filter2D(tmp, tmp, -1, CvContext.gaussianKernel) + CvContext.withGrayCalcHist( + image = tmp, + mask = golden.entityMaskDilated, + ) { hist -> + hist.sumOfHist( + indices = golden.deviationParameters.deviationRange, + maximum = golden.deviationParameters.maximumDeviation, + mapFunc = golden.deviationParameters.deviationFunc::calculate, + ) + } + } + if (deviation <= golden.deviationParameters.maximumDeviation) { + send(Candidate(endpix, dirVec, deviation)) + } + } + + identity.transformedLines.forEach { + launch { + emitDeviation(it.first) + } + launch { + emitDeviation(it.second) + } + } + } + deviationFlow.buffer(COROUTINE_BUFFER_SIZE) + .fold(null) { prev: Candidate?, other: Candidate -> + when { + prev === null -> other + prev.third > other.third -> other + else -> prev + } + } + ?.also { this@bestCenters.send(center to it) } + } + } + } + }.toList().sortedBy { (_, candidate) -> candidate.third } + + // filter all candidates that are too close to each other + return bestCenters + .filterIndexed { index, (center, _) -> + (0 until index).map { bestCenters[it] } + .none { (betterCenter, _) -> + (betterCenter - center).vec2D().magnitude < golden.orientationRadius + } + }.map { golden to it } +} + +private suspend fun findGoldenPoIAnimate( + golden: Golden, + distanceMultiplier: Scalar, + identity: IdentifiableEntity, + centrumMask: Mat, + image: Mat, + displayImage: Mat, + windowMgr: IWindowManager, + parentScope: CoroutineScope, +): List>> { + withTmpMat { tmp -> + identity.distanceMat.convertTo(tmp, CvType.CV_8U) + Core.multiply(tmp, distanceMultiplier, tmp) + displayImage.withSubmatSafe(identity.rect) { target -> + Imgproc.cvtColor(tmp, tmp, Imgproc.COLOR_GRAY2BGR) + tmp.copyTo(target, identity.entityMask) + tmp.setTo(COLOR_BLACK) + tmp.copyTo(target, centrumMask) + Imgproc.cvtColor(centrumMask, tmp, Imgproc.COLOR_GRAY2BGR) + Core.multiply(tmp, golden.goldenColor, tmp, 1.0) + Core.add(tmp, target, target, centrumMask, CvType.CV_8UC3) + } + } + windowMgr.updateImage(displayImage) + delay(ANIMATION_DELAY) + return findGoldenPoI( + image = image, + identity = identity, + golden = golden, + centrumMask = centrumMask + ).also { goldenCandidates -> + val goldenDirection = Vector2D(golden.orientation.vec2D()).normalize() + withContext(SINGLE_DISPATCHER) { + withTmpMat { tmp -> + withTmpMat { displayTmp -> + goldenCandidates.forEach { (golden, centerToCandidate) -> + val (center, candidate) = centerToCandidate + val intoGoldSpace = golden.minMaxDistance.maxLoc.rounded() - center + val rect = golden.rect.apply { + x += identity.rect.x - intoGoldSpace.x + y += identity.rect.y - intoGoldSpace.y + } + image.withSubmatSafe(rect) { subImage -> + subImage.copyTo(displayTmp) + val (endpix, dirVec, deviation) = candidate + val wmg = WINDOW_MANAGER_FACTORY.createWindowManager("$golden on ${identity.rect} with $deviation") + wmg.setDimension(HIST_SIZE, HIST_SIZE) + parentScope.launch { + wmg.display(displayTmp) + } + val angle = acos(dirVec.dot(goldenDirection)) + subImage.copyTo(tmp) + Imgproc.cvtColor(tmp, displayTmp, Imgproc.COLOR_GRAY2BGR) + displayTmp.drawLine( + golden.minMaxDistance.maxLoc.rounded() to intoGoldSpace + endpix, + COLOR_GREEN, + ) + launch { + wmg.updateImage(displayTmp) + } + delay(ANIMATION_DELAY * 1000) + CvContext.withRotationMatrix( + angle = angle, + center = golden.minMaxDistance.maxLoc, + ) { intoGoldSpaceRot -> + Imgproc.warpAffine(tmp, tmp, intoGoldSpaceRot, tmp.size()) + } + Imgproc.cvtColor(tmp, displayTmp, Imgproc.COLOR_GRAY2BGR) + displayTmp.drawLine( + golden.orientation, + COLOR_YELLOW, + ) + launch { + wmg.updateImage(displayTmp) + } + delay(ANIMATION_DELAY * 1000) + Core.absdiff(golden.image, tmp, tmp) + Imgproc.filter2D(tmp, displayTmp, -1, CvContext.gaussianKernel) + launch { + wmg.updateImage(displayTmp) + } + delay(ANIMATION_DELAY) + wmg.updateImage(displayTmp) + delay(10.seconds) + wmg.dispose() + } + } + } + } + } + } +} + +suspend fun main(args: Array) { + CvContext.initialize() + val imageDirectory = Path(args.getOrElse(0) { "." }) + val comparisonData = imageDirectory.resolve("export_data.csv").bufferedReader().lineSequence().drop(1) + .map { line -> line.trimEnd().split(",") } + .groupingBy { parts -> parts[0] } + .aggregateTo(ConcurrentHashMap()) { _, accumulator: MutableList>?, parts, _ -> + val descriptionToRect = parts[5] to Rect( + parts[1].toInt(), + parts[2].toInt(), + parts[3].toInt() - parts[1].toInt() + 1, + parts[4].toInt() - parts[2].toInt() + 1, + ) + (accumulator ?: mutableListOf()).apply { add(descriptionToRect) } + } + val hits = AtomicInteger(0) + val falseHit = AtomicInteger(0) + val count = AtomicInteger(0) + val imagePaths = imageDirectory.listDirectoryEntries("*.bmp") + val perImageWindow = WINDOW_MANAGER_FACTORY.createWindowManager("Current Image") + val scanner = EntityScanner(animate = false, scanBarSpacing = 6, foregroundThreshold = FOREGROUND_THRESHOLD) + + coroutineScope { + val chaetomium = Golden.createGolden( + parentScope = this, + scanner = scanner, + goldenName = "Chaetomium", + goldenColor = COLOR_GREEN, + goldenPath = Resource.getPath("single_chaetomium.bmp"), + deviationParameters = DeviationParameters( + deviationMin = 40, + deviationFunc = { level, value -> + level * (value / 40f) + }, + maximumDeviation = 100.0 + ), + ) + val stachybotry = Golden.createGolden( + parentScope = this, + scanner = scanner, + goldenName = "Stachybotrys", + goldenPath = Resource.getPath("single_stachybotry.bmp"), + goldenColor = COLOR_RED, + deviationParameters = DeviationParameters( + deviationMin = 20, + deviationFunc = { level, value -> + level * (value / 12f) + }, + maximumDeviation = 200.0 + ), + ) + val goldenList = listOf(chaetomium, stachybotry) + val localBufferInfo = when (COROUTINE_BUFFER_SIZE) { + Channel.RENDEZVOUS -> "RENDEZVOUS" + Channel.CONFLATED -> "CONFLATED" + Channel.BUFFERED -> "BUFFERED" + Channel.UNLIMITED -> "UNLIMITED" + else -> toString() + } + println("Coroutine buffer: $localBufferInfo, Parallel images: $MAXIMUM_PARALLEL_IMAGES") + val deferred = async { + val maxDistance = goldenList.maxOf { it.minMaxDistance.maxVal } + val distanceMultiplier = COLOR_WHITE / maxDistance + if (ANIMATION_DELAY.isPositive()) { + goldenList.forEach { + Core.multiply(it.displayImage, distanceMultiplier, it.displayImage) + launch { + it.updateImage() + } + if (ANIMATION_DELAY.isPositive()) delay(ANIMATION_DELAY) + Imgproc.cvtColor(it.image, it.displayImage, Imgproc.COLOR_GRAY2BGR) + it.displayImage.drawLine(it.orientation, COLOR_YELLOW) + it.updateImage() + if (ANIMATION_DELAY.isPositive()) delay(ANIMATION_DELAY) + } + } + + val windowInitialized = AtomicBoolean(false) + val imageFlow = flow { + for (imagePath in imagePaths.shuffled()) { + try { + val (displayImage, image) = CvContext.readAndConvertImage(imagePath) + if (!windowInitialized.getAndSet(true) && ANIMATION_DELAY.isPositive()) { + this@coroutineScope.launch { + try { + perImageWindow.display(displayImage) + } catch (e: CancellationException) { + this@async.cancel("Window was closed", e) + } + } + } + val comparisonSet: MutableList> = comparisonData[imagePath.name] ?: mutableListOf() + emit(Triple(displayImage, image, comparisonSet)) + } catch (error: CvException) { + System.err.println("Cannot read $imagePath: ${error.message}") + } + } + } + imageFlow + .buffer(if (ANIMATION_DELAY.isPositive()) Channel.RENDEZVOUS else MAXIMUM_PARALLEL_IMAGES) + .collect { (displayImage, image, comparison) -> + scanner.scanImage(image, displayImage, perImageWindow).asFlow() + .buffer(COROUTINE_BUFFER_SIZE) + .mapNotNull entityMatching@{ entity -> + withIdEntity(entity) { identity -> + var goldenCandidates = goldenList.flatMap { golden -> + golden.withCentrumMask(identity) { centrumMask -> + if (ANIMATION_DELAY.isPositive()) { + findGoldenPoIAnimate( + golden = golden, + distanceMultiplier = distanceMultiplier, + identity = identity, + centrumMask = centrumMask, + image = image, + displayImage = displayImage, + windowMgr = perImageWindow, + parentScope = this@coroutineScope, + ) + } else { + findGoldenPoI( + image = image, + identity = identity, + golden = golden, + centrumMask = centrumMask, + ) + } + } + }.sortedBy { it.second.second.third } // deviation + goldenCandidates = goldenCandidates + .filterIndexed { index, (_, it) -> + val (center, _) = it + (0 until index).map { goldenCandidates[it] } + .none { (betterGold, other) -> + (other.first - center).vec2D().magnitude < betterGold.orientationRadius + } + } + if (ANIMATION_DELAY.isPositive()) { + goldenCandidates = goldenCandidates.onEach { (gold, it) -> + val (center, candidate) = it + val (_, _, deviation) = candidate + println("Found minimum for ${identity.rect} on $gold: $center (deviation: $deviation)") + Imgproc.drawMarker( + /* img = */ displayImage, + /* position = */ center.vec2D() + identity.rect.tl(), + /* color = */ gold.goldenColor, + /* markerType = */ Imgproc.MARKER_CROSS, + /* markerSize = */ 3, + /* thickness = */ 3, + ) + Imgproc.rectangle(displayImage, identity.rect, gold.goldenColor) + perImageWindow.updateImage(displayImage) + if (ANIMATION_DELAY.isPositive()) delay(ANIMATION_DELAY) + } + } + return@entityMatching goldenCandidates.groupingBy { (gold, _) -> gold.goldenName } + .eachCount() + .maxByOrNull { it.value } + ?.let { goldenAmount -> + val description = if (goldenAmount.value > 1) { + "${goldenAmount.key}_Agglomerate_Candidate" + } else { + "${goldenAmount.key}_Einzelspore_Candidate" + } + Triple(description, identity.rect, comparison) + } + } + } + .collect { result -> + count.incrementAndGet() + val matchingResults = result.third.filter { (description, comparisonRect) -> + description == result.first && + (comparisonRect.contains(result.second.tl()) && + comparisonRect.contains(result.second.br())) || + (result.second.contains(comparisonRect.tl()) && + result.second.contains(comparisonRect.br())) + } + result.third.removeAll(matchingResults) + when (matchingResults.size) { + 0 -> falseHit.incrementAndGet().also { println("False hit ${result.first}") } + 1 -> hits.incrementAndGet().also { println("Hit ${result.first}") } + else -> falseHit.incrementAndGet().also { println("Hit too many ${result.first}") } + } + } + if (ANIMATION_DELAY.isPositive()) delay(ANIMATION_DELAY * 2000) + println("Releasing image $displayImage\n and $image") + displayImage.release() + CvContext.recycle(image) + } + } + deferred.invokeOnCompletion { + perImageWindow.dispose() + goldenList.forEach { it.release() } + if (deferred.isCancelled) { + val error = deferred.getCompletionExceptionOrNull()!! + if (error is CancellationException) { + val cause = error.cause?.takeIf { it.message != error.message }?.let { ". Cause: ".plus(it.message) }.orEmpty() + System.err.println("Cancelled program. Cause: ${error.message}" + cause) + } else { + error.printStackTrace() + } + exitProcess(1) + } + } + deferred.join() + val misses = comparisonData.values.sumOf { it.size } + println("Hits ${hits.get()}, False-hits ${falseHit.get()}, Total ${count.get()}, Misses $misses") + } +} + +internal inline fun Mat.withSubmatSafe(rect: Rect, block: (mat: Mat) -> T): T? { + val submat = try { + submat(rect) + } catch (error: CvException) { + System.err.println("Cannot create submatrix for $this, rect $rect, error:\n $error") + null + } + return submat?.let { mat -> + kotlin.runCatching { block(mat) }.also { mat.release() }.getOrThrow() + } +} + +private fun ClosedFloatingPointRange.progression(step: Number): List = sequence { + var x = start + while (lessThanOrEquals(x, endInclusive)) { + yield(x) + x += step.toDouble() + } +}.toList() diff --git a/src/test/kotlin/network/rs485/ben/computervision/LineTest.kt b/src/test/kotlin/network/rs485/ben/computervision/LineTest.kt new file mode 100644 index 0000000..09b6105 --- /dev/null +++ b/src/test/kotlin/network/rs485/ben/computervision/LineTest.kt @@ -0,0 +1,41 @@ +package network.rs485.ben.computervision + +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +internal class LineTest { + + @Test + fun `line intersect fails for lines on different y`() { + val a = Pixel(0, 0) to Pixel(1, 0) + val b = Pixel(0, 1) to Pixel(1, 1) + assertFalse { a.intersects(b) } + assertFalse { b.intersects(a) } + } + + @Test + fun `line intersect fails for unconnected lines`() { + val a = Pixel(0, 0) to Pixel(1, 0) + val b = Pixel(2, 0) to Pixel(3, 0) + assertFalse { a.intersects(b) } + assertFalse { b.intersects(a) } + } + + @Test + fun `line intersect succeeds for connected lines`() { + val a = Pixel(0, 0) to Pixel(2, 0) + val b = Pixel(1, 0) to Pixel(3, 0) + assertTrue { a.intersects(b) } + assertTrue { b.intersects(a) } + } + + @Test + fun `line intersect succeeds for stacked lines`() { + val a = Pixel(0, 0) to Pixel(3, 0) + val b = Pixel(1, 0) to Pixel(2, 0) + assertTrue { a.intersects(b) } + assertTrue { b.intersects(a) } + } + +}