RGB video input¶

This is an example of applying retinomorphic convolution to an RGB video.

In [1]:

import warnings

warnings.filterwarnings("ignore")

from pathlib import Path
from moviepy import VideoFileClip, TextClip, CompositeVideoClip
from tqdm import tqdm
from moviepy import clips_array, vfx

from matplotlib.font_manager import fontManager
font = fontManager.findfont("monospace")

from pyrception.utils import plot
from pyrception.visual.rf import ReceptiveFields
from pyrception.visual.layers import (
    BipolarLayer,
    AmacrineLayer,
    GanglionLayer,
    ReceptorLayer,
    HorizontalLayer,
)
from pyrception.utils.enums import KernelFilter
from pyrception.utils.processors import VideoLoader

import warnings warnings.filterwarnings("ignore") from pathlib import Path from moviepy import VideoFileClip, TextClip, CompositeVideoClip from tqdm import tqdm from moviepy import clips_array, vfx from matplotlib.font_manager import fontManager font = fontManager.findfont("monospace") from pyrception.utils import plot from pyrception.visual.rf import ReceptiveFields from pyrception.visual.layers import ( BipolarLayer, AmacrineLayer, GanglionLayer, ReceptorLayer, HorizontalLayer, ) from pyrception.utils.enums import KernelFilter from pyrception.utils.processors import VideoLoader

Loading BokehJS ...

Open a video file¶

In [2]:

video_path = Path("./resources/zebra_herd.mp4")
vl = VideoLoader(video_path)
vl.preview()

video_path = Path("./resources/zebra_herd.mp4") vl = VideoLoader(video_path) vl.preview()

©️ Video clip credit: SwissHumanity Stories (licensed under the Pexels terms of use).

Initialise the layers¶

In [3]:

sectors = 128
rc_layer = ReceptorLayer(vl.size)

hz_rfs = ReceptiveFields(
    rc_layer.size,
    kfilter=KernelFilter.Gaussian,
    sectors=sectors,
    kbounds=(15, 15),
)
hz_layer = HorizontalLayer(hz_rfs)

bp_rfs = ReceptiveFields(
    rc_layer.size,
    sectors=sectors,
    substrate=rc_layer.substrate,
)
bp_layer = BipolarLayer(bp_rfs)

am_rfs = ReceptiveFields(
    rc_layer.size,
    sectors=sectors,
    substrate=bp_rfs.cell_coordinates,
    kscale=3.0,
)
am_layer = AmacrineLayer(am_rfs)

sectors = 128 rc_layer = ReceptorLayer(vl.size) hz_rfs = ReceptiveFields( rc_layer.size, kfilter=KernelFilter.Gaussian, sectors=sectors, kbounds=(15, 15), ) hz_layer = HorizontalLayer(hz_rfs) bp_rfs = ReceptiveFields( rc_layer.size, sectors=sectors, substrate=rc_layer.substrate, ) bp_layer = BipolarLayer(bp_rfs) am_rfs = ReceptiveFields( rc_layer.size, sectors=sectors, substrate=bp_rfs.cell_coordinates, kscale=3.0, ) am_layer = AmacrineLayer(am_rfs)

Pyrception | ReceptorLayer | Initialised.
Building kernels...: 100%|██████████| 7229/7229 [00:02<00:00, 2777.03it/s]
Pyrception | HorizontalLayer | Initialised.
Building kernels...: 100%|██████████| 7229/7229 [00:01<00:00, 4363.40it/s]
Pyrception | BipolarLayer | Initialised.
Building kernels...: 100%|██████████| 7229/7229 [00:02<00:00, 3419.19it/s]
Pyrception | AmacrineLayer | Initialised.

In [4]:

center_rfs = ReceptiveFields(
    rc_layer.size,
    sectors=sectors,
    substrate=bp_rfs.cell_coordinates,
)
surround_rfs = ReceptiveFields(
    rc_layer.size,
    sectors=sectors,
    substrate=am_rfs.cell_coordinates,
    kscale=3.0,
)

center_rfs = ReceptiveFields( rc_layer.size, sectors=sectors, substrate=bp_rfs.cell_coordinates, ) surround_rfs = ReceptiveFields( rc_layer.size, sectors=sectors, substrate=am_rfs.cell_coordinates, kscale=3.0, )

Building kernels...: 100%|██████████| 7229/7229 [00:01<00:00, 4429.33it/s]
Building kernels...: 100%|██████████| 7229/7229 [00:02<00:00, 3599.87it/s]

In [5]:

gl_layer = GanglionLayer(
    center_rfs,
    surround_rfs,
    tau=5,
    threshold=0.075,
)

gl_layer = GanglionLayer( center_rfs, surround_rfs, tau=5, threshold=0.075, )

Pyrception | GanglionLayer | Initialised.

Process the video¶

Process the video with all retinal layers and record the output.

In [7]:

vl = VideoLoader(video_path)

with (
    rc_layer.record_raw("./recordings/raw.mp4"),
    hz_layer.record_activations("./recordings/hz_activations.mp4"),
    hz_layer.record_norm("./recordings/hz_norm.mp4"),
    bp_layer.record_on_off("./recordings/bp_on_off.mp4"),
    am_layer.record_activations("./recordings/am_activations.mp4"),
    gl_layer.record_spikes("./recordings/spikes.mp4")
):
    for idx, frame in tqdm(enumerate(vl), total=vl.frame_count):
        rc_out = rc_layer(frame)
        hz_out, hz_fbk = hz_layer(rc_out)
        bp_on, bp_off = bp_layer(rc_out, hz_fbk)
        am_act = am_layer(bp_on)
        spikes = gl_layer(bp_on, am_act)

vl = VideoLoader(video_path) with ( rc_layer.record_raw("./recordings/raw.mp4"), hz_layer.record_activations("./recordings/hz_activations.mp4"), hz_layer.record_norm("./recordings/hz_norm.mp4"), bp_layer.record_on_off("./recordings/bp_on_off.mp4"), am_layer.record_activations("./recordings/am_activations.mp4"), gl_layer.record_spikes("./recordings/spikes.mp4") ): for idx, frame in tqdm(enumerate(vl), total=vl.frame_count): rc_out = rc_layer(frame) hz_out, hz_fbk = hz_layer(rc_out) bp_on, bp_off = bp_layer(rc_out, hz_fbk) am_act = am_layer(bp_on) spikes = gl_layer(bp_on, am_act)

100%|██████████| 756/756 [00:47<00:00, 15.93it/s]

Visualise the responses¶

In [42]:

# A simple captioning wrapper.
def caption(clip: VideoFileClip, text: str) -> CompositeVideoClip:
    txt = TextClip(
        font,
        text,
        font_size=16,
        size=clip.size,
        duration=clip.duration,
        stroke_color="#000",
        stroke_width=1,
        color="#eee",
        vertical_align="top",
    )
    return CompositeVideoClip([clip, txt])

# A simple captioning wrapper. def caption(clip: VideoFileClip, text: str) -> CompositeVideoClip: txt = TextClip( font, text, font_size=16, size=clip.size, duration=clip.duration, stroke_color="#000", stroke_width=1, color="#eee", vertical_align="top", ) return CompositeVideoClip([clip, txt])

In [18]:

rc_clip = VideoFileClip(video_path).without_audio()
hz_act_clip = VideoFileClip("./recordings/hz_activations.mp4").without_audio()
hz_norm_clip = VideoFileClip("./recordings/hz_norm.mp4").without_audio()
bp_on_off_clip = VideoFileClip("./recordings/bp_on_off.mp4").without_audio()
am_clip = VideoFileClip("./recordings/am_activations.mp4").without_audio()
gl_clip = VideoFileClip("./recordings/spikes.mp4").without_audio()

rc_clip = VideoFileClip(video_path).without_audio() hz_act_clip = VideoFileClip("./recordings/hz_activations.mp4").without_audio() hz_norm_clip = VideoFileClip("./recordings/hz_norm.mp4").without_audio() bp_on_off_clip = VideoFileClip("./recordings/bp_on_off.mp4").without_audio() am_clip = VideoFileClip("./recordings/am_activations.mp4").without_audio() gl_clip = VideoFileClip("./recordings/spikes.mp4").without_audio()

Create a 3x2 composite array of clips.

In [33]:

comp = clips_array(
    [
        [
            caption(rc_clip, "Original"),
            caption(hz_act_clip, "Horizontal activations"),
        ],
        [
            caption(hz_norm_clip, "Normalised"),
            caption(bp_on_off_clip, "ON (red) / OFF (green) bipolar"),
        ],
        [
            caption(am_clip, "Amacrine"),
            caption(gl_clip, "Ganglion"),
        ],
    ]
)

comp = clips_array( [ [ caption(rc_clip, "Original"), caption(hz_act_clip, "Horizontal activations"), ], [ caption(hz_norm_clip, "Normalised"), caption(bp_on_off_clip, "ON (red) / OFF (green) bipolar"), ], [ caption(am_clip, "Amacrine"), caption(gl_clip, "Ganglion"), ], ] )

In [ ]:

# Play only the first 8 seconds.
comp.display_in_notebook(rd_kwargs={"logger": None})

# Play only the first 8 seconds. comp.display_in_notebook(rd_kwargs={"logger": None})

Out[ ]: