change model loading, add measure, update requriements.txt

Author: Gerald

.gitignore

@@ -1,3 +1,4 @@
 __pycache__
 *.pyc
 *~
+xcuserdata

Pytorch-CoreML-Spectrogram.xcodeproj/project.pbxproj

@@ -426,7 +426,7 @@
 				CODE_SIGN_STYLE = Automatic;
 				DEVELOPMENT_TEAM = K6KE3UBBT4;
 				INFOPLIST_FILE = "Pytorch-CoreML-SpectrogramTests/Info.plist";
-				IPHONEOS_DEPLOYMENT_TARGET = 13.4;
+				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
@@ -448,7 +448,7 @@
 				CODE_SIGN_STYLE = Automatic;
 				DEVELOPMENT_TEAM = K6KE3UBBT4;
 				INFOPLIST_FILE = "Pytorch-CoreML-SpectrogramTests/Info.plist";
-				IPHONEOS_DEPLOYMENT_TARGET = 13.4;
+				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",

Pytorch-CoreML-Spectrogram/ViewController.swift

@@ -19,7 +19,7 @@ class ViewController: UIViewController {
     // set up for audio
     private let audioEngine = AVAudioEngine()
     // specify the audio samples format the CoreML model
-    let desiredAudioFormat: AVAudioFormat = {
+    private let desiredAudioFormat: AVAudioFormat = {
         let avAudioChannelLayout = AVAudioChannelLayout(layoutTag: kAudioChannelLayoutTag_Mono)!
         return AVAudioFormat(
             commonFormat: .pcmFormatFloat32,
@@ -30,10 +30,12 @@ class ViewController: UIViewController {
     }()
 
     // create a queue to do analysis on a separate thread
-    let analysisQueue = DispatchQueue(label: "com.myco.AnalysisQueue")
+    private let analysisQueue = DispatchQueue(label: "com.myco.AnalysisQueue")
 
     // instantiate our model
-    let model = wave__melspec()
+
+
+    var model : wave__melspec? = nil
     typealias NetworkInput = wave__melspecInput
     typealias NetworkOutput = wave__melspecOutput
 
@@ -46,12 +48,24 @@ class ViewController: UIViewController {
     override func viewDidLoad() {
         super.viewDidLoad()
         // Do any additional setup after loading the view.
+        load_model()
     }
 
     override func viewDidAppear(_ animated: Bool) {
         startAudioEngine()
     }
 
+    private func load_model() {
+        let config = MLModelConfiguration()
+        config.computeUnits = .all
+        do {
+            self.model = try wave__melspec( configuration: config )
+        } catch {
+            fatalError( "unable to load ML model!" )
+        }
+
+    }
+    
     // audio capture via microphone
     private func startAudioEngine() {
 
@@ -151,7 +165,8 @@ class ViewController: UIViewController {
 
 
     func predict_provider(provider: MLDictionaryFeatureProvider ) {
-        if let outFeatures = try? self.model.model.prediction(from: provider) {
+
+        if let outFeatures = try? self.model?.model.prediction(from: provider) {
             // release the semaphore as soon as the model is done
             self.semaphore.signal()
 

Pytorch-CoreML-SpectrogramTests/Pytorch_CoreML_SpectrogramTests.swift

@@ -151,10 +151,40 @@ class Pytorch_CoreML_SpectrogramTests: XCTestCase {
 
     }
 
-    func testPerformanceExample() throws {
+    func test_inference_time() throws {
         // This is an example of a performance test case.
+        let model = wave__melspec()
+
+        let array_shape: [NSNumber] = [1, 12800]
+        let audioData = try! MLMultiArray(shape: array_shape, dataType: MLMultiArrayDataType.float32 )
+        let inputs: [String: Any] = [
+            "input.1": audioData,
+        ]
+        // container for ML Model inputs
+        let provider = try! MLDictionaryFeatureProvider(dictionary: inputs)
+
         self.measure {
             // Put the code you want to measure the time of here.
+            let N = 100
+            let start_time = CACurrentMediaTime()
+            let options = MLPredictionOptions()
+            // options.usesCPUOnly = true
+            for _ in 0..<N {
+                _ = try? model.model.prediction(
+                    from: provider,
+                    options: options
+                )
+            }
+            let elapsed = CACurrentMediaTime() - start_time
+            print( "avg inference time: \(elapsed/Double(N))")
+            /* simulator:
+                avg inference time: 0.011592097150278279
+                w/ CPUOnly: avg inference time: 0.011968133399495855
+               on iPhone XR
+                avg inference time: 0.003219694583094679
+                w/ CPUOnly: avg inference time: 0.003034873333526775
+
+             */
         }
     }
 

python/model.py

@@ -130,7 +130,7 @@ def convert_to_coreml( self, fn_mlmodel, sample_input, plot_specs=True ):
         assert torch_output.shape == mlmodel_output.shape
 
         print( 'sum diff ', np.sum( np.abs( torch_output-mlmodel_output) ), np.max( np.abs( torch_output-mlmodel_output) ) )
-        # !!!!!!!!! assert np.allclose( torch_output, mlmodel_output, atol=1e-3, rtol=1e-5 )
+        assert np.allclose( torch_output, mlmodel_output, atol=2, rtol=2 ) # big tolerance due to log scale
 
         print( 'Successful MLModel conversion to %s!' % fn_mlmodel )
 

python/requirements.txt

@@ -1,6 +1,7 @@
 torch
 librosa
-torchlibrosa
+# torchlibrosa
+-e git://github.com/ml-illustrated/torchlibrosa.git#egg=torchlibrosa
 SoundFile
 onnx
 onnxruntime