[POC- DO NOT MERGE] Use heatmap to fingerprint recording data

package.json

@@ -54,6 +54,7 @@
     "@microsoft/applicationinsights-web": "^3.0.0",
     "@sentry/svelte": "^7.100.1",
     "@tensorflow/tfjs": "^4.4.0",
+    "@tensorflow/tfjs-vis": "^1.5.1",
     "@types/w3c-web-serial": "^1.0.6",
     "@types/w3c-web-usb": "^1.0.6",
     "bowser": "^2.11.0",

src/__tests__/ml.test.ts

@@ -8,22 +8,26 @@
  */
 
 import * as tf from '@tensorflow/tfjs';
-import { makeInputs, trainModel } from '../script/ml';
+import { ModelSettings, makeInputs, trainModel } from '../script/ml';
 import { gestures } from '../script/stores/Stores';
 import gestureData from './fixtures/gesture-data.json';
 import gestureDataBadLabels from './fixtures/gesture-data-bad-labels.json';
 import testdataShakeStill from './fixtures/test-data-shake-still.json';
 import { PersistantGestureData } from '../script/domain/Gestures';
+import { get } from 'svelte/store';
+import { settings } from '../script/stores/mlStore';
 
 let tensorFlowModel: tf.LayersModel | void;
+const mlSettings = get(settings);
+const modelSettings = {
+  axes: mlSettings.includedAxes,
+  filters: mlSettings.includedFilters,
+};
+
 beforeAll(async () => {
   // No webgl in tests running in node.
   tf.setBackend('cpu');
 
-  // This creates determinism in the model training step.
-  const randomSpy = vi.spyOn(Math, 'random');
-  randomSpy.mockImplementation(() => 0.5);
-
   gestures.importFrom(gestureData);
   tensorFlowModel = await trainModel();
 });
@@ -34,7 +38,7 @@ const getModelResults = (data: PersistantGestureData[]) => {
   const numActions = data.length;
   data.forEach((action, index) => {
     action.recordings.forEach(recording => {
-      x.push(makeInputs(recording.data));
+      x.push(makeInputs(modelSettings, recording.data));
       const label = new Array(numActions);
       label.fill(0, 0, numActions);
       label[index] = 1;
@@ -91,8 +95,8 @@ describe('Model tests', () => {
 
   test('returns correct results on testing data', async () => {
     const { tensorFlowResultAccuracy } = getModelResults(testdataShakeStill);
-    // The model thinks two samples of still are circle.
-    // 14 samples; 1.0 / 14 = 0.0714; 0.0714 * 12 correct inferences = 0.8571
-    expect(parseFloat(tensorFlowResultAccuracy)).toBeGreaterThan(0.85);
+    // The model thinks one sample of still is circle.
+    // 14 samples; 1.0 / 14 = 0.0714; 0.0714 * 13 correct inferences = 0.9286
+    expect(parseFloat(tensorFlowResultAccuracy)).toBeGreaterThan(0.9);
   });
 });

src/components/AverageFingerprint.svelte

@@ -0,0 +1,50 @@
+<!--
+  (c) 2024, Center for Computational Thinking and Design at Aarhus University and contributors
+
+  SPDX-License-Identifier: MIT
+ -->
+
+<script lang="ts">
+  import { get } from 'svelte/store';
+  import { makeInputs, ModelSettings } from '../script/ml';
+  import { GestureData, settings } from '../script/stores/mlStore';
+  import Fingerprint from './Fingerprint.svelte';
+
+  export let gesture: GestureData;
+
+  const filters = Array.from(get(settings).includedFilters);
+
+  const mlSettings = get(settings);
+  const modelSettings: ModelSettings = {
+    axes: mlSettings.includedAxes,
+    filters: mlSettings.includedFilters,
+  };
+
+  const filtersLabels: string[] = [];
+  filters.forEach(filter => {
+    filtersLabels.push(`${filter}-x`, `${filter}-y`, `${filter}-z`);
+  });
+
+  const getFingerprintData = () => {
+    const inputData: number[][] = [];
+    gesture.recordings.forEach(r => {
+      inputData.push(makeInputs(modelSettings, r.data, 'computeNormalizedOutput'));
+    });
+    const averagedData: number[] = [];
+    for (let i = 0; i < filters.length * 3; i++) {
+      let filterValues: number[] = [];
+      inputData.forEach(d => {
+        filterValues.push(d[i]);
+      });
+      averagedData.push(filterValues.reduce((a, b) => a + b, 0) / filterValues.length);
+    }
+    return averagedData;
+  };
+</script>
+
+<div class="h-full overflow-hidden">
+  <Fingerprint
+    gestureName={`${gesture.name} average`}
+    recordingData={undefined}
+    averagedData={getFingerprintData()} />
+</div>

src/components/Fingerprint.svelte

@@ -0,0 +1,66 @@
+<!--
+  (c) 2024, Center for Computational Thinking and Design at Aarhus University and contributors
+
+  SPDX-License-Identifier: MIT
+ -->
+
+<script lang="ts">
+  import * as tfvis from '@tensorflow/tfjs-vis';
+  import { onMount } from 'svelte';
+  import { get } from 'svelte/store';
+  import { makeInputs, ModelSettings } from '../script/ml';
+  import { RecordingData, settings } from '../script/stores/mlStore';
+
+  export let recordingData: RecordingData | undefined;
+  export let gestureName: string;
+  export let averagedData: number[] = [];
+
+  let surface: undefined | tfvis.Drawable;
+
+  const mlSettings = get(settings);
+  const modelSettings: ModelSettings = {
+    axes: mlSettings.includedAxes,
+    filters: mlSettings.includedFilters,
+  };
+
+  const filtersLabels: string[] = [];
+  modelSettings.filters.forEach(filter => {
+    filtersLabels.push(`${filter}-x`, `${filter}-y`, `${filter}-z`);
+  });
+
+  const getProcessedData = () =>
+    recordingData
+      ? makeInputs(modelSettings, recordingData.data, 'computeNormalizedOutput')
+      : [];
+
+  const chartData = {
+    values: recordingData ? [getProcessedData()] : [averagedData],
+    xTickLabels: [gestureName],
+    yTickLabels: filtersLabels,
+  };
+
+  onMount(() => {
+    if (surface) {
+      tfvis.render.heatmap(surface, chartData, {
+        colorMap: 'viridis',
+        height: 109,
+        width: 80,
+        domain: [0, 1],
+        fontSize: 0,
+      });
+    }
+  });
+</script>
+
+<div class="relative w-40px" class:h-full={!recordingData}>
+  <div
+    class="absolute h-full w-full -left-10px right-0 -bottom-1px"
+    class:top-1px={!recordingData}
+    class:top-0={recordingData}>
+    <div bind:this={surface}></div>
+  </div>
+  {#if !recordingData}
+    <div class="absolute bg-white h-full w-20px left-34px top-0 bottom-0" />
+    <div class="absolute bg-white h-8px w-40px left-0 -bottom-7px" />
+  {/if}
+</div>

src/components/Gesture.svelte

@@ -164,12 +164,12 @@
   }
 
   // Delete recording from recordings array
-  function deleteRecording(recording: RecordingData) {
+  function deleteRecording(gestureId: number, recording: RecordingData) {
     if (!areActionsAllowed(false)) {
       return;
     }
     $state.isPredicting = false;
-    removeRecording(gesture.getId(), recording.ID);
+    removeRecording(gestureId, recording.ID);
   }
 
   function selectGesture(): void {
@@ -366,7 +366,12 @@
         </div>
         {#if hasRecordings}
           {#each $gesture.recordings as recording (String($gesture.ID) + String(recording.ID))}
-            <Recording {recording} onDelete={deleteRecording} on:focus={selectGesture} />
+            <Recording
+              gestureId={$gesture.ID}
+              gestureName={$nameBind}
+              {recording}
+              onDelete={deleteRecording}
+              on:focus={selectGesture} />
           {/each}
         {/if}
       </div>

src/components/GestureTilePart.svelte

@@ -13,7 +13,8 @@
 </script>
 
 <div
-  class="{$$restProps.class || ''} rounded-lg bg-backgroundlight border-1 {selected
+  class="{$$restProps.class ||
+    ''} overflow-hidden rounded-lg bg-backgroundlight border-1 {selected
     ? 'border-brand-500'
     : 'border-transparent'}"
   class:h-30={small}

src/components/Recording.svelte

@@ -5,25 +5,41 @@
  -->
 
 <script lang="ts">
-  import { fade } from 'svelte/transition';
+  import CloseIcon from 'virtual:icons/ri/close-line';
+  import { t } from '../i18n';
   import type { RecordingData } from '../script/stores/mlStore';
+  import Fingerprint from './Fingerprint.svelte';
   import RecordingGraph from './graphs/RecordingGraph.svelte';
   import IconButton from './IconButton.svelte';
-  import { t } from '../i18n';
-  import CloseIcon from 'virtual:icons/ri/close-line';
 
   // get recording from mother prop
   export let recording: RecordingData;
-  export let onDelete: (recording: RecordingData) => void;
+  export let gestureName: string;
+  export let gestureId: number;
+  export let showFingerprint: boolean = false;
+  export let onDelete: (gestureId: number, recording: RecordingData) => void;
+  export let fullWidth: boolean = false;
+  export let showProcessedData: boolean = false;
 </script>
 
-<div class="h-full w-40 relative">
-  <RecordingGraph data={recording.data} />
+<div
+  class="h-full flex w-40 relative overflow-hidden"
+  class:w-40={!fullWidth}
+  class:w-full={fullWidth}
+  class:rounded-md={showFingerprint}
+  class:border-1={showFingerprint}
+  class:border-neutral-300={showFingerprint}>
+  <RecordingGraph data={recording.data} showBorder={!showFingerprint} />
+  {#if showFingerprint}
+    <div class="transition-all duration-1000 w-0" class:w-30px={showProcessedData}>
+      <Fingerprint {gestureName} recordingData={recording} />
+    </div>
+  {/if}
 
-  <div class="absolute right-0 top-0 z-2">
+  <div class="absolute top-0 z-2 left-0">
     <IconButton
       ariaLabel={$t('content.data.deleteRecording')}
-      onClick={() => onDelete(recording)}
+      onClick={() => onDelete(gestureId, recording)}
       on:focus>
       <CloseIcon class="text-xl m-1" />
     </IconButton>

src/components/bottom/BottomPanel.svelte

@@ -5,15 +5,18 @@
  -->
 
 <script lang="ts">
-  import LiveGraph from '../graphs/LiveGraph.svelte';
+  import { onMount } from 'svelte';
   import { t } from '../../i18n';
-  import ConnectedLiveGraphButtons from './ConnectedLiveGraphButtons.svelte';
-  import LiveGraphInformationSection from './LiveGraphInformationSection.svelte';
-  import BaseDialog from '../dialogs/BaseDialog.svelte';
+  import { state } from '../../script/stores/uiStore';
   import View3DLive from '../3d-inspector/View3DLive.svelte';
+  import BaseDialog from '../dialogs/BaseDialog.svelte';
+  import LiveGraph from '../graphs/LiveGraph.svelte';
   import Information from '../information/Information.svelte';
-  import { state } from '../../script/stores/uiStore';
+  import ConnectedLiveGraphButtons from './ConnectedLiveGraphButtons.svelte';
+  import LiveFingerprint from './LiveFingerprint.svelte';
+  import LiveGraphInformationSection from './LiveGraphInformationSection.svelte';
 
+  export let showFingerprint: boolean = false;
   const live3dViewVisible = false;
   const live3dViewSize = live3dViewVisible ? 160 : 0;
   let componentWidth: number;
@@ -23,7 +26,9 @@
 <div bind:clientWidth={componentWidth} class="relative w-full h-full bg-white">
   <div class="relative z-1">
     <div
-      class="flex items-center justify-between gap-2 pt-4 px-7 m-0 absolute top-0 left-0 right-0">
+      class="flex items-center justify-between gap-2 pt-4 px-7 m-0 absolute top-0 left-0"
+      class:right-0={!showFingerprint}
+      class:right-80px={showFingerprint}>
       <div class="flex items-center gap-4">
         <!-- The live text and info box -->
         <LiveGraphInformationSection />
@@ -63,7 +68,11 @@
       </BaseDialog>
     {/if}
   </div>
-  <div class="absolute w-full h-full">
-    <LiveGraph width={componentWidth - live3dViewSize} />
+  <div class="absolute w-full h-full overflow-hidden" class:flex={showFingerprint}>
+    <!-- <div class="bg-red-200" style={`width: ${componentWidth}px`}></div> -->
+    <LiveGraph width={componentWidth - live3dViewSize - (showFingerprint ? 80 : 0)} />
+    {#if showFingerprint}
+      <LiveFingerprint />
+    {/if}
   </div>
 </div>

src/components/bottom/LiveFingerprint.svelte

@@ -0,0 +1,64 @@
+<!--
+  (c) 2024, Center for Computational Thinking and Design at Aarhus University and contributors
+
+  SPDX-License-Identifier: MIT
+ -->
+
+<script lang="ts">
+  import * as tfvis from '@tensorflow/tfjs-vis';
+  import { onMount } from 'svelte';
+  import { get } from 'svelte/store';
+  import { makeInputs, ModelSettings } from '../../script/ml';
+  import { getPrevData, settings } from '../../script/stores/mlStore';
+  import { state } from '../../script/stores/uiStore';
+
+  let surface: undefined | tfvis.Drawable;
+  $: prevData = getPrevData();
+  const filters = Array.from(get(settings).includedFilters);
+
+  const mlSettings = get(settings);
+  const modelSettings: ModelSettings = {
+    axes: mlSettings.includedAxes,
+    filters: mlSettings.includedFilters,
+  };
+
+  const filtersLabels: string[] = [];
+  filters.forEach(filter => {
+    filtersLabels.push(`${filter}-x`, `${filter}-y`, `${filter}-z`);
+  });
+
+  const disconnectedData = new Array(filtersLabels.length).fill(0);
+
+  onMount(() => {
+    const interval = setInterval(() => {
+      prevData = getPrevData();
+
+      if (surface) {
+        const currentGestureData = prevData
+          ? makeInputs(modelSettings, prevData, 'computeNormalizedOutput')
+          : disconnectedData;
+        const currentData = {
+          values: [currentGestureData],
+          xTickLabels: ['Live'],
+          yTickLabels: filtersLabels,
+        };
+        tfvis.render.heatmap(surface, currentData, {
+          colorMap: 'viridis',
+          height: 166,
+          width: 160,
+          domain: [0, 1],
+          fontSize: 0,
+        });
+      }
+    });
+    return () => {
+      clearInterval(interval);
+    };
+  });
+</script>
+
+<div class="relative w-20 h-full z-0">
+  <div class="absolute h-full w-full top-0 -left-10px right-0 bottom-0">
+    <div bind:this={surface}></div>
+  </div>
+</div>