(ns cortex.vision "Simulate the sense of vision in jMonkeyEngine3. Enables multiple eyes from different positions to observe the same world, and pass the observed data to any arbitrary function. Automatically reads eye-nodes from specially prepared blender files and instantiates them in the world as actual eyes." {:author "Robert McIntyre"} (:use (cortex world sense util)) (:import com.jme3.post.SceneProcessor) (:import (com.jme3.util BufferUtils Screenshots)) (:import java.nio.ByteBuffer) (:import java.awt.image.BufferedImage) (:import (com.jme3.renderer ViewPort Camera)) (:import (com.jme3.math ColorRGBA Vector3f Matrix3f)) (:import com.jme3.renderer.Renderer) (:import com.jme3.app.Application) (:import com.jme3.texture.FrameBuffer) (:import (com.jme3.scene Node Spatial))) (defn vision-pipeline "Create a SceneProcessor object which wraps a vision processing continuation function. The continuation is a function that takes [#^Renderer r #^FrameBuffer fb #^ByteBuffer b #^BufferedImage bi], each of which has already been appropriately sized." [continuation] (let [byte-buffer (atom nil) renderer (atom nil) image (atom nil)] (proxy [SceneProcessor] [] (initialize [renderManager viewPort] (let [cam (.getCamera viewPort) width (.getWidth cam) height (.getHeight cam)] (reset! renderer (.getRenderer renderManager)) (reset! byte-buffer (BufferUtils/createByteBuffer (* width height 4))) (reset! image (BufferedImage. width height BufferedImage/TYPE_4BYTE_ABGR)))) (isInitialized [] (not (nil? @byte-buffer))) (reshape [_ _ _]) (preFrame [_]) (postQueue [_]) (postFrame [#^FrameBuffer fb] (.clear @byte-buffer) (continuation @renderer fb @byte-buffer @image)) (cleanup [])))) (defn frameBuffer->byteBuffer! "Transfer the data in the graphics card (Renderer, FrameBuffer) to the CPU (ByteBuffer)." [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb] (.readFrameBuffer r fb bb) bb) (defn byteBuffer->bufferedImage! "Convert the C-style BGRA image data in the ByteBuffer bb to the AWT style ABGR image data and place it in BufferedImage bi." [#^ByteBuffer bb #^BufferedImage bi] (Screenshots/convertScreenShot bb bi) bi) (defn BufferedImage! "Continuation which will grab the buffered image from the materials provided by (vision-pipeline)." [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb #^BufferedImage bi] (byteBuffer->bufferedImage! (frameBuffer->byteBuffer! r fb bb) bi)) (defn retina-sensor-profile "Return a map of pixel sensitivity numbers to BufferedImages describing the distribution of light-sensitive components of this eye. :red, :green, :blue, :gray are already defined as extracting the red, green, blue, and average components respectively." [#^Spatial eye] (if-let [eye-map (meta-data eye "eye")] (map-vals load-image (eval (read-string eye-map))))) (defn eye-dimensions "Returns [width, height] determined by the metadata of the eye." [#^Spatial eye] (let [dimensions (map #(vector (.getWidth %) (.getHeight %)) (vals (retina-sensor-profile eye)))] [(apply max (map first dimensions)) (apply max (map second dimensions))])) (in-ns 'cortex.vision) (defn add-eye! "Create a Camera centered on the current position of 'eye which follows the closest physical node in 'creature. The camera will point in the X direction and use the Z vector as up as determined by the rotation of these vectors in blender coordinate space. Use XZY rotation for the node in blender." [#^Node creature #^Spatial eye] (let [target (closest-node creature eye) [cam-width cam-height] ;;[640 480] ;; graphics card on laptop doesn't support ;; arbitray dimensions. (eye-dimensions eye) cam (Camera. cam-width cam-height) rot (.getWorldRotation eye)] (.setLocation cam (.getWorldTranslation eye)) (.lookAtDirection cam ; this part is not a mistake and (.mult rot Vector3f/UNIT_X) ; is consistent with using Z in (.mult rot Vector3f/UNIT_Y)) ; blender as the UP vector. (.setFrustumPerspective cam (float 45) (float (/ (.getWidth cam) (.getHeight cam))) (float 1) (float 1000)) (bind-sense target cam) cam)) (def sensitivity-presets "Retinal sensitivity presets for sensors that extract one channel (:red :blue :green) or average all channels (:all)" {:all 0xFFFFFF :red 0xFF0000 :blue 0x0000FF :green 0x00FF00}) (def ^{:doc "Return the children of the creature's \"eyes\" node." :arglists '([creature])} eyes (sense-nodes "eyes")) (in-ns 'cortex.vision) (defn add-camera! "Add a camera to the world, calling continuation on every frame produced." [#^Application world camera continuation] (let [width (.getWidth camera) height (.getHeight camera) render-manager (.getRenderManager world) viewport (.createMainView render-manager "eye-view" camera)] (doto viewport (.setClearFlags true true true) (.setBackgroundColor ColorRGBA/Black) (.addProcessor (vision-pipeline continuation)) (.attachScene (.getRootNode world))))) (in-ns 'cortex.vision) (defrecord attached-viewport [vision-fn viewport-fn] clojure.lang.IFn (invoke [this world] (vision-fn world)) (applyTo [this args] (apply vision-fn args))) (defn pixel-sense [sensitivity pixel] (let [s-r (bit-shift-right (bit-and 0xFF0000 sensitivity) 16) s-g (bit-shift-right (bit-and 0x00FF00 sensitivity) 8) s-b (bit-and 0x0000FF sensitivity) p-r (bit-shift-right (bit-and 0xFF0000 pixel) 16) p-g (bit-shift-right (bit-and 0x00FF00 pixel) 8) p-b (bit-and 0x0000FF pixel) total-sensitivity (* 255 (+ s-r s-g s-b))] (float (/ (+ (* s-r p-r) (* s-g p-g) (* s-b p-b)) total-sensitivity)))) (defn vision-kernel "Returns a list of functions, each of which will return a color channel's worth of visual information when called inside a running simulation." [#^Node creature #^Spatial eye & {skip :skip :or {skip 0}}] (let [retinal-map (retina-sensor-profile eye) camera (add-eye! creature eye) vision-image (atom (BufferedImage. (.getWidth camera) (.getHeight camera) BufferedImage/TYPE_BYTE_BINARY)) register-eye! (runonce (fn [world] (add-camera! world camera (let [counter (atom 0)] (fn [r fb bb bi] (if (zero? (rem (swap! counter inc) (inc skip))) (reset! vision-image (BufferedImage! r fb bb bi))))))))] (vec (map (fn [[key image]] (let [whites (white-coordinates image) topology (vec (collapse whites)) sensitivity (sensitivity-presets key key)] (attached-viewport. (fn [world] (register-eye! world) (vector topology (vec (for [[x y] whites] (pixel-sense sensitivity (.getRGB @vision-image x y)))))) register-eye!))) retinal-map)))) (defn gen-fix-display "Create a function to call to restore a simulation's display when it is disrupted by a Viewport." [] (runonce (fn [world] (add-camera! world (.getCamera world) no-op)))) (defn vision! "Returns a list of functions, each of which returns visual sensory data when called inside a running simulation." [#^Node creature & {skip :skip :or {skip 0}}] (reduce concat (for [eye (eyes creature)] (vision-kernel creature eye)))) (in-ns 'cortex.vision) (defn view-vision "Creates a function which accepts a list of visual sensor-data and displays each element of the list to the screen." [] (view-sense (fn [[coords sensor-data]] (let [image (points->image coords)] (dorun (for [i (range (count coords))] (.setRGB image ((coords i) 0) ((coords i) 1) (gray (int (* 255 (sensor-data i))))))) image))))