Add Weighted k-means (#66)

Author: ishii-norimi

README.md

@@ -121,7 +121,7 @@ for (let i = 0; i < n; i++) {
 
 | task | model |
 | ---- | ----- |
-| clustering | k-means, k-means++, k-medois, k-medians, x-means, G-means, LBG, ISODATA, Soft k-means, Fuzzy c-means, Possibilistic c-means, Kernel k-means, Agglomerative (complete linkage, single linkage, group average, Ward's, centroid, weighted average, median), DIANA, Monothetic, Mean shift, DBSCAN, OPTICS, HDBSCAN, DENCLUE, PAM, CLARA, CLARANS, BIRCH, CURE, ROCK, PLSA, Latent dirichlet allocation, GMM, VBGMM, Affinity propagation, Spectral clustering, Mountain, SOM, GTM, (Growing) Neural gas, Growing cell structures, LVQ, ART, SVC, CAST, NMF, Autoencoder |
+| clustering | k-means, k-means++, k-medois, k-medians, x-means, G-means, Weighted k-means, LBG, ISODATA, Soft k-means, Fuzzy c-means, Possibilistic c-means, Kernel k-means, Agglomerative (complete linkage, single linkage, group average, Ward's, centroid, weighted average, median), DIANA, Monothetic, Mean shift, DBSCAN, OPTICS, HDBSCAN, DENCLUE, PAM, CLARA, CLARANS, BIRCH, CURE, ROCK, PLSA, Latent dirichlet allocation, GMM, VBGMM, Affinity propagation, Spectral clustering, Mountain, SOM, GTM, (Growing) Neural gas, Growing cell structures, LVQ, ART, SVC, CAST, NMF, Autoencoder |
 | classification | Linear discriminant (FLD, LDA), Quadratic discriminant, Mixture discriminant, Least squares, Ridge, (Complement / Negation / Universal-set / Selective) Naive Bayes (gaussian), AODE, k-nearest neighbor, Radius neighbor, Fuzzy k-nearest neighbor, Nearest centroid, DANN, Decision tree, Random forest, Extra trees, GBDT, XGBoost, ALMA, ROMMA, Online gradient descent, Passive aggressive, RLS, Second order perceptron, AROW, NAROW, Confidence weighted, CELLIP, IELLIP, Normal herd, (Multinomial) Logistic regression, (Multinomial) Probit, SVM, Gaussian process, HMM, CRF, Bayesian Network, LVQ, Perceptron, ADALINE, MLP, LMNN |
 | semi-supervised classification | k-nearest neighbor, Radius neighbor, Label propagation, Label spreading, k-means, GMM, Ladder network |
 | regression | Least squares, Ridge, Lasso, Elastic net, RLS, Bayesian linear, Poisson, Least absolute deviations, Huber, Tukey, Least trimmed squares, Least median squares, Lp norm linear, SMA, Deming, Segmented, LOWESS, spline, Gaussian process, Principal components, Partial least squares, Projection pursuit, Quantile regression, k-nearest neighbor, Radius neighbor, IDW, Nadaraya Watson, Priestley Chao, Gasser Muller, RBF Network, RVM, Decision tree, Random forest, Extra trees, GBDT, XGBoost, SVR, MLP, GMR, Isotonic, Ramer Douglas Peucker, Theil-Sen, Passing-Bablok, Repeated median |

js/model_selector.js

@@ -51,6 +51,7 @@ const AIMethods = [
 				{ value: 'kmeans', title: 'K-Means(++) / K-Medoids / K-Medians' },
 				{ value: 'xmeans', title: 'X-Means' },
 				{ value: 'gmeans', title: 'G-Means' },
+				{ value: 'weighted_kmeans', title: 'Weighted k-means' },
 				{ value: 'isodata', title: 'ISODATA' },
 				{ value: 'soft_kmeans', title: 'Soft K-Means' },
 				{ value: 'fuzzy_cmeans', title: 'Fuzzy C-Means' },

js/view/weighted_kmeans.js

@@ -0,0 +1,85 @@
+import WeightedKMeans from '../../lib/model/weighted_kmeans.js'
+
+var dispWKMeans = function (elm, platform) {
+	let model = null
+
+	elm.append('span').text('beta')
+	elm.append('input')
+		.attr('name', 'beta')
+		.attr('type', 'number')
+		.attr('min', 1)
+		.attr('max', 10)
+		.attr('step', 0.1)
+		.attr('value', 2)
+	const slbConf = platform.setting.ml.controller.stepLoopButtons().init(() => {
+		platform.init()
+		const beta = +elm.select('[name=beta]').property('value')
+		model = new WeightedKMeans(beta)
+		elm.select('[name=clusternumber]').text(model.size + ' clusters')
+	})
+	elm.append('input')
+		.attr('type', 'button')
+		.attr('value', 'Add centroid')
+		.on('click', () => {
+			platform.fit((tx, ty, pred_cb) => {
+				model.add(tx)
+				const pred = model.predict(tx)
+				pred_cb(pred.map(v => v + 1))
+			})
+			platform.centroids(
+				model.centroids,
+				model.centroids.map((c, i) => i + 1),
+				{ line: true }
+			)
+			elm.select('[name=clusternumber]').text(model.size + ' clusters')
+		})
+	elm.append('span').attr('name', 'clusternumber').style('padding', '0 10px').text('0 clusters')
+
+	slbConf.step(cb => {
+		if (model.size === 0) {
+			cb && cb()
+			return
+		}
+		platform.fit((tx, ty, pred_cb) => {
+			model.fit(
+				tx,
+				ty.map(v => v[0])
+			)
+			const pred = model.predict(tx)
+			pred_cb(pred.map(v => v + 1))
+		})
+		platform.centroids(
+			model.centroids,
+			model.centroids.map((c, i) => i + 1),
+			{
+				line: true,
+				duration: 1000,
+			}
+		)
+		cb && setTimeout(cb, 1000)
+	})
+	elm.append('input')
+		.attr('type', 'button')
+		.attr('value', 'Skip')
+		.on('click', () => {
+			platform.fit((tx, ty, pred_cb) => {
+				ty = ty.map(v => v[0])
+				while (model.fit(tx, ty) > 1.0e-8);
+				const pred = model.predict(tx)
+				pred_cb(pred.map(v => v + 1))
+			})
+			platform.centroids(
+				model.centroids,
+				model.centroids.map((c, i) => i + 1),
+				{
+					line: true,
+					duration: 1000,
+				}
+			)
+		})
+}
+
+export default function (platform) {
+	platform.setting.ml.usage = 'Click and add data point. Then, click "Step" button repeatedly.'
+	dispWKMeans(platform.setting.ml.configElement, platform)
+}

lib/model/weighted_kmeans.js

@@ -0,0 +1,143 @@
+/**
+ * Weighted k-means model
+ */
+export default class WeightedKMeans {
+	// http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.715.8143&rep=rep1&type=pdf#page=124
+	// https://www.ijert.org/research/a-comparative-study-of-k-means-and-weighted-k-means-for-clustering-IJERTV1IS10227.pdf
+	/**
+	 * @param {number} beta
+	 */
+	constructor(beta) {
+		this._beta = beta
+		this._centroids = []
+		this._w = null
+	}
+
+	/**
+	 * Centroids
+	 *
+	 * @type {Array<Array<number>>}
+	 */
+	get centroids() {
+		return this._centroids
+	}
+
+	/**
+	 * Number of clusters.
+	 *
+	 * @type {number}
+	 */
+	get size() {
+		return this._centroids.length
+	}
+
+	_distance(a, b) {
+		let v = 0
+		for (let i = a.length - 1; i >= 0; i--) {
+			v += this._w[i] ** this._beta * (a[i] - b[i]) ** 2
+		}
+		return v
+	}
+
+	/**
+	 * Add a new cluster.
+	 *
+	 * @param {Array<Array<number>>} datas
+	 * @returns {number[]}
+	 */
+	add(datas) {
+		if (!this._w) {
+			this._w = Array(datas[0].length).fill(1 / datas[0].length)
+		}
+		while (true) {
+			const p = datas[Math.floor(Math.random() * datas.length)]
+			if (Math.min(...this._centroids.map(c => c.reduce((s, v, k) => s + (v - p[k]) ** 2, 0))) > 1.0e-8) {
+				const cpoint = p.concat()
+				this._centroids.push(cpoint)
+				return cpoint
+			}
+		}
+	}
+
+	/**
+	 * Clear all clusters.
+	 */
+	clear() {
+		this._centroids = []
+		this._w = null
+	}
+
+	/**
+	 * Returns predicted categories.
+	 *
+	 * @param {Array<Array<number>>} datas
+	 * @returns {number[]}
+	 */
+	predict(datas) {
+		if (this._centroids.length === 0) {
+			return
+		}
+		return datas.map(value => {
+			let min_d = Infinity
+			let min_k = -1
+			for (let i = 0; i < this._centroids.length; i++) {
+				const d = this._distance(value, this._centroids[i])
+				if (d < min_d) {
+					min_d = d
+					min_k = i
+				}
+			}
+			return min_k
+		})
+	}
+
+	/**
+	 * Fit model and returns total distance the centroid has moved.
+	 *
+	 * @param {Array<Array<number>>} datas
+	 * @returns {number}
+	 */
+	fit(datas) {
+		if (this._centroids.length === 0 || datas.length === 0) {
+			return 0
+		}
+		if (!this._w) {
+			this._w = Array(datas[0].length).fill(1 / datas[0].length)
+		}
+		const oldCentroids = this._centroids
+
+		const pred = this.predict(datas)
+		this._centroids = this._centroids.map((c, k) => {
+			const m = Array(datas[0].length).fill(0)
+			let s = 0
+			for (let i = 0; i < datas.length; i++) {
+				if (pred[i] !== k) {
+					continue
+				}
+				for (let j = 0; j < m.length; j++) {
+					m[j] += datas[i][j]
+				}
+				s++
+			}
+			return m.map(v => v / s)
+		})
+
+		const newpred = this.predict(datas)
+		const d = Array(this._w.length).fill(0)
+		for (let i = 0; i < datas.length; i++) {
+			for (let j = 0; j < d.length; j++) {
+				d[j] += (datas[i][j] - this._centroids[newpred[i]][j]) ** 2
+			}
+		}
+		for (let k = 0; k < d.length; k++) {
+			let v = 0
+			for (let j = 0; j < d.length; j++) {
+				v += (d[k] / d[j]) ** (1 / (this._beta - 1))
+			}
+			this._w[k] = 1 / v
+		}
+
+		const err = oldCentroids.reduce((s, c, i) => s + this._distance(c, this._centroids[i]), 0)
+		return err
+	}
+}

tests/lib/model/weighted_kmeans.test.js

@@ -0,0 +1,28 @@
+import Matrix from '../../../lib/util/matrix.js'
+import WeightedKMeans from '../../../lib/model/weighted_kmeans.js'
+
+import { randIndex } from '../../../lib/evaluate/clustering.js'
+
+test('predict', () => {
+	const model = new WeightedKMeans(2)
+	const n = 50
+	const x = Matrix.randn(n, 2, 0, 0.1).concat(Matrix.randn(n, 2, 5, 0.1)).toArray()
+
+	model.add(x)
+	model.add(x)
+	for (let i = 0; i < 20; i++) {
+		const d = model.fit(x)
+		if (d === 0) {
+			break
+		}
+	}
+	const y = model.predict(x)
+	expect(y).toHaveLength(x.length)
+
+	const t = []
+	for (let i = 0; i < x.length; i++) {
+		t[i] = Math.floor(i / n)
+	}
+	const ri = randIndex(y, t)
+	expect(ri).toBeGreaterThan(0.9)
+})