Support multiple n-gram lengths

Author: dexyk

README.md

@@ -2,7 +2,15 @@
 
 The plan for this package is to have Go implementation of different string distance/similarity functions, like Levenshtein (normalized, weighted, Damerau), Jaro-Winkler, Jaccard index, Euclidean distance, Hamming distance...
 
-Currently it has implemented Levenshtein, Jaccard, Hamming, LCS, Q-gram and Cosine distance functions. Work in progress...
+Currently it has implemented:
+ - Levenshtein
+ - Jaccard
+ - Hamming
+ - LCS
+ - Q-gram
+ - n-gram based Cosine distanc
+
+ Work in progress...
 
 ## Import and installation
 
@@ -30,7 +38,7 @@ Currently only Levenshtein, Jaccard, Hamming, LCS string, Q-gram and Cosine dist
 
 #### Levenshtein
 
-Levenshtein distance can be calculated with default parameters (use DefaultSimilarityOptions) where cost of insert, delete and substitute operation are 1. You can also use it with other parameters by using SimilarityOptions type. Setting CaseInsensitive to true in SimilarityOptions the comparison will be done without considering character cases. 
+Levenshtein distance can be calculated with default parameters (use DefaultSimilarityOptions) where cost of insert, delete and substitute operation are 1. You can also use it with other parameters by using SimilarityOptions type. Setting CaseInsensitive to true in SimilarityOptions the comparison will be done without considering character cases.
 
 Example:
 
@@ -62,9 +70,9 @@ Example:
 ```go
     fmt.Println(stringosim.Jaccard([]rune("stringosim"), []rune("stingobim")))
 
-    fmt.Println(stringosim.Jaccard([]rune("stringosim"), []rune("stingobim"), 2))
+    fmt.Println(stringosim.Jaccard([]rune("stringosim"), []rune("stingobim"), []int{2}))
 
-    fmt.Println(stringosim.Jaccard([]rune("stringosim"), []rune("stingobim"), 3))
+    fmt.Println(stringosim.Jaccard([]rune("stringosim"), []rune("stingobim"), []int{3}))
 ```
 
 #### Hamming
@@ -97,7 +105,7 @@ Example:
 ```go
     fmt.Println(stringosim.LCS([]rune("testing lcs algorithm"), []rune("another l c s example")))
 
-    fmt.Println(stringosim.LCS([]rune("testing lcs algorithm"), []rune("ANOTHER L C S EXAMPLE"), 
+    fmt.Println(stringosim.LCS([]rune("testing lcs algorithm"), []rune("ANOTHER L C S EXAMPLE"),
     stringosim.LCSSimilarityOptions{
         CaseInsensitive: true,
     }))
@@ -139,7 +147,7 @@ Example:
     fmt.Println(stringosim.QGram([]rune("abcde"), []rune("ABDCDE"),
     stringosim.QGramSimilarityOptions{
         CaseInsensitive: true,
-        NGramLength:     3,
+        NGramSizes:     []int{3},
     }))
 ```
 
@@ -155,7 +163,6 @@ Example:
     fmt.Println(stringosim.Cosine(Cosine[]rune("abcde"), []rune("ABDCDE"),
     stringosim.CosineSimilarityOptions{
         CaseInsensitive: true,
-        NGramLength:     3,
+        NGramSizes:     []int{3},
     }))
 ```
-

cosine.go

@@ -1,43 +1,43 @@
 package stringosim
 
 import (
-    "strings"
+	"strings"
 )
 
 type CosineSimilarityOptions struct {
-    CaseInsensitive bool
-    NGramLength     int
+	CaseInsensitive bool
+	NGramSizes      []int
 }
 
 var DefaultCosineSimilarityOptions = CosineSimilarityOptions{
-    CaseInsensitive: false,
-    NGramLength:     2,
+	CaseInsensitive: false,
+	NGramSizes:      []int{2},
 }
 
 func Cosine(s []rune, t []rune, options ...CosineSimilarityOptions) float64 {
-    if len(s) == 0 {
-        if len(t) == 0 {
-            return float64(0.0)
-        } else {
-            return float64(1.0)
-        }
-    } else {
-        if len(t) == 0 {
-            return float64(1.0)
-        }
-    }
-    opt := DefaultCosineSimilarityOptions
-    for _, option := range options {
-        opt = option
-        break
-    }
-    var sGrams, tGrams map[string]int
-    if opt.CaseInsensitive {
-        sGrams = GetNGram(strings.ToLower(string(s)), opt.NGramLength)
-        tGrams = GetNGram(strings.ToLower(string(t)), opt.NGramLength)
-    } else {
-        sGrams = GetNGram(string(s), opt.NGramLength)
-        tGrams = GetNGram(string(t), opt.NGramLength)
-    }
-    return 1.0 - float64(DotProductNGrams(sGrams, tGrams))/NormNGram(sGrams)/NormNGram(tGrams)
+	if len(s) == 0 {
+		if len(t) == 0 {
+			return float64(0.0)
+		} else {
+			return float64(1.0)
+		}
+	} else {
+		if len(t) == 0 {
+			return float64(1.0)
+		}
+	}
+	opt := DefaultCosineSimilarityOptions
+	for _, option := range options {
+		opt = option
+		break
+	}
+	var sGrams, tGrams map[string]int
+	if opt.CaseInsensitive {
+		sGrams = GetNGram(strings.ToLower(string(s)), opt.NGramSizes)
+		tGrams = GetNGram(strings.ToLower(string(t)), opt.NGramSizes)
+	} else {
+		sGrams = GetNGram(string(s), opt.NGramSizes)
+		tGrams = GetNGram(string(t), opt.NGramSizes)
+	}
+	return 1.0 - float64(DotProductNGrams(sGrams, tGrams))/NormNGram(sGrams)/NormNGram(tGrams)
 }

cosine_test.go

@@ -1,41 +1,41 @@
 package stringosim
 
 import (
-    "testing"
+	"testing"
 )
 
 var testCosineOptions1 CosineSimilarityOptions = CosineSimilarityOptions{
-    CaseInsensitive: true,
-    NGramLength:     3,
+	CaseInsensitive: true,
+	NGramSizes:      []int{3},
 }
 
 type CosineTest struct {
-    src string
-    trg string
-    dis float64
-    opt CosineSimilarityOptions
+	src string
+	trg string
+	dis float64
+	opt CosineSimilarityOptions
 }
 
 var CosineTests = []CosineTest{
-    {"", "", 0.0, DefaultCosineSimilarityOptions},
-    {"xxxyyy", "xxxyyy", 0.0, DefaultCosineSimilarityOptions},
-    {"xxxyyy", "yyyxxx", 0.1111111111111, DefaultCosineSimilarityOptions},
-    {"xxyzxyyzzy", "xyyxzyzxyzyx", 0.2364582844290667, DefaultCosineSimilarityOptions},
-    {"xxyzxyyzzy", "XYYXZYZXYZYX", 0.5527864045000421, testCosineOptions1},
-    {"xxyyzz", "xxxzzz", 0.40371520600005606, DefaultCosineSimilarityOptions},
-    {"asdlkajsdlkasdkj", "fkdsjlkdf", 0.8825559560970593, DefaultCosineSimilarityOptions},
-    {"STRING", "sting", 1.0, DefaultCosineSimilarityOptions},
-    {"STRING", "sting", 0.7113248654051871, testCosineOptions1},
-    {"comparing the string similarity", "this is compared using Cosine similarity", 0.3519132720916943, DefaultCosineSimilarityOptions},
-    {"comparing the string similarity", "this is compared using Cosine similarity", 0.5291024827582208, testCosineOptions1},
+	{"", "", 0.0, DefaultCosineSimilarityOptions},
+	{"xxxyyy", "xxxyyy", 0.0, DefaultCosineSimilarityOptions},
+	{"xxxyyy", "yyyxxx", 0.1111111111111, DefaultCosineSimilarityOptions},
+	{"xxyzxyyzzy", "xyyxzyzxyzyx", 0.2364582844290667, DefaultCosineSimilarityOptions},
+	{"xxyzxyyzzy", "XYYXZYZXYZYX", 0.5527864045000421, testCosineOptions1},
+	{"xxyyzz", "xxxzzz", 0.40371520600005606, DefaultCosineSimilarityOptions},
+	{"asdlkajsdlkasdkj", "fkdsjlkdf", 0.8825559560970593, DefaultCosineSimilarityOptions},
+	{"STRING", "sting", 1.0, DefaultCosineSimilarityOptions},
+	{"STRING", "sting", 0.7113248654051871, testCosineOptions1},
+	{"comparing the string similarity", "this is compared using Cosine similarity", 0.3519132720916943, DefaultCosineSimilarityOptions},
+	{"comparing the string similarity", "this is compared using Cosine similarity", 0.5291024827582208, testCosineOptions1},
 }
 
 func TestCosine(t *testing.T) {
-    for _, test := range CosineTests {
-        dis := Cosine([]rune(test.src), []rune(test.trg), test.opt)
-        if !EqualFloat64(dis, test.dis) {
-            t.Log("Cosine distance between", test.src, "and", test.trg, "is", dis, "but should be", test.dis)
-            t.Fail()
-        }
-    }
+	for _, test := range CosineTests {
+		dis := Cosine([]rune(test.src), []rune(test.trg), test.opt)
+		if !EqualFloat64(dis, test.dis) {
+			t.Log("Cosine distance between", test.src, "and", test.trg, "is", dis, "but should be", test.dis)
+			t.Fail()
+		}
+	}
 }

helper.go

@@ -30,25 +30,19 @@ func EqualFloat64(x, y float64) bool {
 	return math.Abs(x-y) < EPS
 }
 
-func GetNGram(s string, n ...int) map[string]int {
-	nGram := 1
-	if len(n) > 0 {
-		for _, v := range n {
-			nGram = v
-			break
-		}
-	}
-
+func GetNGram(s string, NGramSizes []int) map[string]int {
 	regExp := regexp.MustCompile(`\s+`)
 	t := regExp.ReplaceAllString(s, " ")
 	m := make(map[string]int)
-	for i := 0; i <= len(t)-nGram; i++ {
-		v := string(t[i:(i + nGram)])
-		cnt, ok := m[v]
-		if ok {
-			m[v] = cnt + 1
-		} else {
-			m[v] = 1
+	for _, nGram := range NGramSizes {
+		for i := 0; i <= len(t)-nGram; i++ {
+			v := string(t[i:(i + nGram)])
+			cnt, ok := m[v]
+			if ok {
+				m[v] = cnt + 1
+			} else {
+				m[v] = 1
+			}
 		}
 	}
 	return m

jaccard.go

@@ -1,25 +1,16 @@
 package stringosim
 
-import ()
+func Jaccard(s []rune, t []rune, NGramSizes []int) float64 {
+	sGrams := GetNGram(string(s), NGramSizes)
+	tGrams := GetNGram(string(t), NGramSizes)
 
-func Jaccard(s []rune, t []rune, n ...int) float64 {
-    nGram := 1
-    if len(n) > 0 {
-        for _, v := range n {
-            nGram = v
-            break
-        }
-    }
-    sGrams := GetNGram(string(s), nGram)
-    tGrams := GetNGram(string(t), nGram)
-
-    total := len(sGrams) + len(tGrams)
-    intersection := 0
-    for k, _ := range sGrams {
-        _, ok := tGrams[k]
-        if ok {
-            intersection++
-        }
-    }
-    return 1.0 - float64(intersection)/float64(total-intersection)
+	total := len(sGrams) + len(tGrams)
+	intersection := 0
+	for k, _ := range sGrams {
+		_, ok := tGrams[k]
+		if ok {
+			intersection++
+		}
+	}
+	return 1.0 - float64(intersection)/float64(total-intersection)
 }

jaccard_test.go

@@ -1,39 +1,39 @@
 package stringosim
 
 import (
-    "testing"
+	"testing"
 )
 
 type JaccardTest struct {
-    src   string
-    trg   string
-    nGram int
-    dis   float64
+	src        string
+	trg        string
+	NGramSizes []int
+	dis        float64
 }
 
 var jaccardTests = []JaccardTest{
-    {"abracadabra", "baccarda", 1, 0.0},
-    {"abracadabra", "baccarda", 2, 0.7272727272727273},
-    {"abracadabra", "baccarda", 3, 1.0},
-    {"this    is     space    test", "this is space test", 3, 0.0},
-    {"just another test of jaccard", "i will test jaccard", 1, 0.47058823529411764},
-    {"just another test of jaccard", "i will test jaccard", 2, 0.6129032258064516},
-    {"just another test of jaccard", "i will test jaccard", 3, 0.6875},
-    {"book is on the shelv", "buk is on the shelf", 1, 0.2142857142857143},
-    {"book is on the shelv", "buk is on the shelf", 2, 0.33333333333333337},
-    {"book is on the shelv", "buk is on the shelf", 3, 0.33333333333333337},
-    {"cardiogram", "krdiogram", 1, 0.2222222222222222},
-    {"cardiogram", "krdiogram", 2, 0.30000000000000004},
-    {"cardiogram", "krdiogram", 3, 0.33333333333333337},
+	{"abracadabra", "baccarda", []int{1}, 0.0},
+	{"abracadabra", "baccarda", []int{2}, 0.7272727272727273},
+	{"abracadabra", "baccarda", []int{3}, 1.0},
+	{"this    is     space    test", "this is space test", []int{3}, 0.0},
+	{"just another test of jaccard", "i will test jaccard", []int{1}, 0.47058823529411764},
+	{"just another test of jaccard", "i will test jaccard", []int{2}, 0.6129032258064516},
+	{"just another test of jaccard", "i will test jaccard", []int{3}, 0.6875},
+	{"book is on the shelv", "buk is on the shelf", []int{1}, 0.2142857142857143},
+	{"book is on the shelv", "buk is on the shelf", []int{2}, 0.33333333333333337},
+	{"book is on the shelv", "buk is on the shelf", []int{3}, 0.33333333333333337},
+	{"cardiogram", "krdiogram", []int{1}, 0.2222222222222222},
+	{"cardiogram", "krdiogram", []int{2}, 0.30000000000000004},
+	{"cardiogram", "krdiogram", []int{3}, 0.33333333333333337},
 }
 
 func TestJaccard(t *testing.T) {
-    for _, test := range jaccardTests {
-        dis := Jaccard([]rune(test.src), []rune(test.trg), test.nGram)
-        if !EqualFloat64(dis, test.dis) {
-            t.Log("Jaccard distance between", test.src, "and", test.trg, "is", dis, "but should be", test.dis)
-            t.Fail()
-        }
+	for _, test := range jaccardTests {
+		dis := Jaccard([]rune(test.src), []rune(test.trg), test.NGramSizes)
+		if !EqualFloat64(dis, test.dis) {
+			t.Log("Jaccard distance between", test.src, "and", test.trg, "is", dis, "but should be", test.dis)
+			t.Fail()
+		}
 
-    }
+	}
 }