Add HLLE (#40)

Author: ishii-norimi

README.md

@@ -127,7 +127,7 @@ for (let i = 0; i < n; i++) {
 | regression | Least squares, Ridge, Lasso, Elastic net, RLS, Bayesian linear, Poisson, Least absolute deviations, Huber, Tukey, Least trimmed squares, Least median squares, Lp norm linear, SMA, Deming, Segmented, LOWESS, spline, Gaussian process, Principal components, Partial least squares, Projection pursuit, Quantile regression, k-nearest neighbor, Radius neighbor, IDW, Nadaraya Watson, Priestley Chao, Gasser Muller, RBF Network, RVM, Decision tree, Random forest, Extra trees, GBDT, XGBoost, SVR, MLP, GMR, Isotonic, Ramer Douglas Peucker, Theil-Sen, Passing-Bablok, Repeated median |
 | interpolation | Nearest neighbor, IDW, Linear, Spherical linear, Brahmagupta, Logarithmic, Cosine, (Inverse) Smoothstep, Cubic, (Centripetal) Catmull-Rom, Hermit, Polynomial, Lagrange, Trigonometric, Spline, RBF Network, Akima, Natural neighbor, Delaunay |
 | anomaly detection | Percentile, MAD, Tukey's fences, Grubbs's test, Thompson test, Tietjen Moore test, Generalized ESD, Hotelling, MT, MCD, k-nearest neighbor, LOF, PCA, OCSVM, KDE, GMM, Isolation forest, Autoencoder, GAN |
-| dimensionality reduction | Random projection, (Dual/Kernel) PCA, Incremental PCA, Probabilistic PCA, GPLVM, LSA, MDS, Linear discriminant analysis, NCA, ICA, Principal curve, Sammon, FastMap, Sliced inverse regression, LLE, Laplacian eigenmaps, Isomap, Diffusion map, SNE, t-SNE, UMAP, SOM, GTM, NMF, MOD, Autoencoder, VAE |
+| dimensionality reduction | Random projection, (Dual/Kernel) PCA, Incremental PCA, Probabilistic PCA, GPLVM, LSA, MDS, Linear discriminant analysis, NCA, ICA, Principal curve, Sammon, FastMap, Sliced inverse regression, LLE, HLLE, Laplacian eigenmaps, Isomap, Diffusion map, SNE, t-SNE, UMAP, SOM, GTM, NMF, MOD, Autoencoder, VAE |
 | feature selection | Mutual information, Ridge, Lasso, Elastic net, Decision tree, NCA |
 | transformation | Box-Cox, Yeo-Johnson |
 | density estimation | Histogram, Average shifted histogram, Polynomial histogram, Maximum likelihood, Kernel density estimation, k-nearest neighbor, Naive Bayes, GMM, HMM |

js/model_selector.js

@@ -299,6 +299,7 @@ const AIMethods = [
 			{ value: 'fastmap', title: 'FastMap' },
 			{ value: 'sir', title: 'Sliced Inverse Regression' },
 			{ value: 'lle', title: 'LLE' },
+			{ value: 'hlle', title: 'HLLE' },
 			{ value: 'laplacian_eigenmaps', title: 'Laplacian eigenmaps' },
 			{ value: 'isomap', title: 'Isomap' },
 			{ value: 'diffusion_map', title: 'Diffusion map' },

js/view/hlle.js

@@ -0,0 +1,29 @@
+import HLLE from '../../lib/model/hlle.js'
+
+var dispHLLE = function (elm, platform) {
+	const fitModel = () => {
+		platform.fit((tx, ty, pred_cb) => {
+			const neighbor = +elm.select('[name=neighbor_size]').property('value')
+			const dim = platform.dimension
+			const y = new HLLE(neighbor).predict(tx, dim)
+			pred_cb(y)
+		})
+	}
+
+	elm.append('span')
+		.text('Select neighbor #')
+		.append('input')
+		.attr('type', 'number')
+		.attr('name', 'neighbor_size')
+		.attr('value', 20)
+		.attr('min', 1)
+	elm.append('input')
+		.attr('type', 'button')
+		.attr('value', 'Fit')
+		.on('click', () => fitModel())
+}
+
+export default function (platform) {
+	platform.setting.ml.usage = 'Click and add data point. Next, click "Fit" button.'
+	dispHLLE(platform.setting.ml.configElement, platform)
+}

lib/model/hlle.js

@@ -0,0 +1,92 @@
+import Matrix from '../util/matrix.js'
+
+/**
+ * Hessian Locally Linear Embedding
+ */
+export default class HLLE {
+	// https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.3.673&rep=rep1&type=pdf
+	// https://github.com/gdkrmr/dimRed
+	// https://github.com/scikit-learn/scikit-learn/blob/7e1e6d09bcc2eaeba98f7e737aac2ac782f0e5f1/sklearn/manifold/_locally_linear.py#L344
+	/**
+	 * @param {number} [k=1]
+	 */
+	constructor(k = 1) {
+		this._k = k
+	}
+
+	/**
+	 * Returns reduced values.
+	 *
+	 * @param {Array<Array<number>>} x
+	 * @param {number} [rd=0]
+	 * @returns {Array<Array<number>>}
+	 */
+	predict(x, rd = 0) {
+		x = Matrix.fromArray(x)
+		const d = x.cols
+		const n = x.rows
+
+		const distance = []
+		const neighbors = []
+		for (let i = 0; i < n; distance[i++] = []);
+		for (let i = 0; i < n; i++) {
+			for (let j = i + 1; j < n; j++) {
+				let dt = 0
+				for (let k = 0; k < d; k++) {
+					dt += (x.at(i, k) - x.at(j, k)) ** 2
+				}
+				distance[i][j] = distance[j][i] = dt
+			}
+
+			const nns = []
+			for (let j = 0; j < n; j++) {
+				if (j === i) continue
+				const dt = distance[i][j]
+				if (nns.length < this._k || dt < nns[this._k - 1].dt) {
+					if (nns.length === this._k) nns.pop()
+					nns.push({
+						dt: dt,
+						idx: j,
+					})
+					for (let k = nns.length - 1; k > 0; k--) {
+						if (nns[k].dt < nns[k - 1].dt) {
+							;[nns[k], nns[k - 1]] = [nns[k - 1], nns[k]]
+						}
+					}
+				}
+			}
+			neighbors.push(nns)
+		}
+
+		const m = Matrix.eye(n, n)
+		for (let i = 0; i < n; i++) {
+			const z = x.row(neighbors[i].map(v => v.idx))
+			z.sub(z.mean(0))
+
+			const [u] = z.svd()
+			const yi = Matrix.ones(this._k, 1 + rd + (rd * (rd + 1)) / 2)
+			yi.set(0, 1, u.slice(0, rd, 1))
+
+			let j = 1 + rd
+			for (let k = 0; k < rd; k++) {
+				yi.set(0, j, u.col(k).copyMult(u.slice(k, rd, 1)))
+				j += rd - k
+			}
+
+			const [q] = yi.qr()
+			const w = q.slice(rd + 1, null, 1)
+			w.div(w.sum(0))
+			const wwt = w.dot(w.t)
+
+			for (let k = 0; k < this._k; k++) {
+				for (let l = 0; l < this._k; l++) {
+					m.addAt(neighbors[i][k].idx, neighbors[i][l].idx, wwt.at(k, l))
+				}
+			}
+		}
+
+		const ev = m.eigenVectors()
+		ev.flip(1)
+		return ev.slice(1, rd + 1, 1).toArray()
+	}
+}

tests/lib/model/hlle.test.js

@@ -0,0 +1,14 @@
+import { jest } from '@jest/globals'
+jest.retryTimes(3)
+
+import Matrix from '../../../lib/util/matrix.js'
+import HLLE from '../../../lib/model/hlle.js'
+
+import { coRankingMatrix } from '../../../lib/evaluate/dimensionality_reduction.js'
+
+test('dimensionality reduction', () => {
+	const x = Matrix.randn(50, 5, 0, 0.2).concat(Matrix.randn(50, 5, 5, 0.2)).toArray()
+	const y = new HLLE(10).predict(x, 2)
+	const q = coRankingMatrix(x, y, 20, 20)
+	expect(q).toBeGreaterThan(0.9)
+})