Update to add support for captcha with english letter, implement 5/6 digits classifier and data augmentation.

Author: JasonLiTW

.gitignore

@@ -113,3 +113,4 @@ venv.bak/
 /data
 /test
 /logs
+/developing

README.md

@@ -2,7 +2,9 @@
 from random import randint
 import csv
 import numpy as np
-FONTPATH = ["./data/times-bold.ttf", "./data/courier-bold.ttf"]
+FONTPATH = ["./data/font/times-bold.ttf", "./data/font/courier-bold.ttf"]
+ENGSTR = "ABCDEFGHJKLMNPQRSTUVWXYZ" # 沒有O和I
+LETTERSTR = "0123456789ABCDEFGHJKLMNPQRSTUVWXYZ"
 
 
 class rect:
@@ -41,27 +43,35 @@ def draw(self, image, overlay):
 
 
 class captchatext:
-    def __init__(self, priority, offset):
-        self.number = randint(0, 9)
+    def __init__(self, priority, offset, captchalen, engletter, ENGNOLIMIT):
+        self.engletter = engletter
+        if ENGNOLIMIT:
+            engletter = True if randint(1, 34) <= 24 else False
+        if engletter:
+            self.letter = ENGSTR[randint(0, len(ENGSTR) - 1)]
+        else:
+            self.letter = str(randint(0, 9))
         self.color = [randint(10, 140) for _ in range(3)]
         self.angle = randint(-55, 55)
         self.priority = priority
         self.offset = offset
         self.next_offset = 0
+        self.captchalen = captchalen
 
 
     def draw(self, image):
         color = (self.color[0], self.color[1], self.color[2], 255)
         font = ImageFont.truetype(FONTPATH[randint(0, 1)], randint(25, 27) * 10)
-        text = Image.new("RGBA", (150, 300), (0, 0, 0, 0))
+        text = Image.new("RGBA", (font.getsize(self.letter)[0], 300), (0, 0, 0, 0))
         textdraw = ImageDraw.Draw(text)
-        textdraw.text((0, 0), str(self.number), font=font, fill=color)
+        textdraw.text((0, 0), self.letter, font=font, fill=color)
         text = text.rotate(self.angle, expand=True)
         text = text.resize((int(text.size[0] / 10), int(text.size[1] / 10)))
-        base = int(self.priority * (200 / 6))
-        rand_min = (offset - base - 2) if (offset - base - 2) >= -15 else -15
+        base = int(self.priority * (200 / self.captchalen))
+        rand_min = (self.offset - base - 4) if (self.offset - base - 4) >= -15 else -15
         rand_min = 0 if self.priority == 0 else rand_min
-        rand_max = (33 - text.size[0]) if self.priority == 5 else (33 - text.size[0] + 10)
+        avg_dp = int(200 / self.captchalen)
+        rand_max = (avg_dp - text.size[0]) if self.priority == self.captchalen - 1 else (avg_dp - text.size[0] + 10)
         try:
             displace = randint(rand_min, rand_max)
         except:
@@ -71,34 +81,43 @@ def draw(self, image):
         image.paste(text, location, text)
 
 
-outputcsv = open('./data/train_set/train.csv', 'w', encoding = 'utf8', newline = '')
-numberlist = []
-status = 1
-for index in range(1, 50001, 1):
-    numberstr = ""
-    bgcolor = [randint(180, 250) for _ in range(3)]
-    captcha = Image.new('RGBA', (200, 60), (bgcolor[0], bgcolor[1], bgcolor[2], 255))
-    rectlist = [rect() for _ in range(32)]
-    for obj in rectlist:
-        obj.draw(image=captcha, overlay=False)
+def generate(GENNUM, SAVEPATH, ENGP=25, FIVEP=0, ENGNOLIMIT=False, filename="train"):
+    captchacsv = open(SAVEPATH + "captcha_{:s}.csv".format(filename), 'w', encoding = 'utf8', newline = '')
+    lencsv = open(SAVEPATH + "len_{:s}.csv".format(filename), 'w', encoding = 'utf8', newline = '')
+    letterlist = []
+    lenlist = []
+    for index in range(1, GENNUM + 1, 1):
+        captchastr = ""
+        captchalen = 5 if randint(1, 100) <= FIVEP else 6
+        engat = randint(0, captchalen - 1) if randint(1, 100) <= ENGP else -1
+        bgcolor = [randint(180, 250) for _ in range(3)]
+        captcha = Image.new('RGBA', (200, 60), (bgcolor[0], bgcolor[1], bgcolor[2], 255))
+        rectlist = [rect() for _ in range(32)]
+        for obj in rectlist:
+            obj.draw(image=captcha, overlay=False)
+        offset = 0
+        for i in range(captchalen):
+            newtext = captchatext(i, offset, captchalen, (True if engat == i else False), ENGNOLIMIT)
+            newtext.draw(image=captcha)
+            offset = newtext.next_offset
+            captchastr += str(newtext.letter)
+        letterlist.append([str(index), captchastr])
+        lenlist.append([str(index), captchalen])
+        for obj in rectlist:
+            obj.draw(image=captcha, overlay=True)
+        captcha.convert("RGB").save(SAVEPATH + str(index).zfill(len(str(GENNUM))) + ".jpg", "JPEG")
+    writer = csv.writer(captchacsv)
+    writer.writerows(letterlist)
+    writer = csv.writer(lencsv)
+    writer.writerows(lenlist)
+    captchacsv.close()
+    lencsv.close()
 
-    offset = 0
-    for i in range(6):
-        newtext = captchatext(i, offset)
-        newtext.draw(image=captcha)
-        offset = newtext.next_offset
-        numberstr += str(newtext.number)
-    numberlist.append([str(index), numberstr])
 
-    for obj in rectlist:
-        obj.draw(image=captcha, overlay=True)
-
-    captcha.convert("RGB").save("./data/train_set/" + str(index) + ".jpg", "JPEG")
-
-    if (index / 50000) >= (status * 0.01):
-        print("...." + str(status) + "%")
-        status += 1
-
-writer = csv.writer(outputcsv)
-writer.writerows(numberlist)
-outputcsv.close()
+if __name__ == "__main__":
+    generate(50000, "./data/56_imitate_train_set/",  ENGP=100, FIVEP=50, ENGNOLIMIT=True, filename="train")
+    generate(10240, "./data/56_imitate_vali_set/",  ENGP=100, FIVEP=50, ENGNOLIMIT=True, filename="vali")
+    generate(50000, "./data/5_imitate_train_set/",  ENGP=100, FIVEP=100, ENGNOLIMIT=True, filename="train")
+    generate(10240, "./data/5_imitate_vali_set/",  ENGP=100, FIVEP=100, ENGNOLIMIT=True, filename="vali")
+    generate(50000, "./data/6_imitate_train_set/",  ENGP=100, FIVEP=0, ENGNOLIMIT=True, filename="train")
+    generate(10240, "./data/6_imitate_vali_set/",  ENGP=100, FIVEP=0, ENGNOLIMIT=True, filename="vali")

captcha_gen.py

@@ -0,0 +1,11 @@
+import shutil
+import requests
+import time
+SAVEPATH = "./data/manual_label/"
+url = 'http://railway1.hinet.net/ImageOut.jsp'
+for i in range(1, 3000):
+    response = requests.get(url, stream=True)
+    with open(SAVEPATH + str(i) + '.jpg', 'wb') as out_file:
+        shutil.copyfileobj(response.raw, out_file)
+    del response
+    time.sleep(0.5)

captcha_scrawl.py

@@ -0,0 +1,25 @@
+from keras.preprocessing.image import ImageDataGenerator
+from PIL import Image
+import numpy as np
+import csv
+
+outputcsv = open('./data/6_real_train_set/captcha_train.csv', 'w', encoding = 'utf8', newline = '') # 輸出csv
+inputcsv = open('./data/manual_label/captcha_vali.csv', 'r', encoding = 'utf8')
+data = [np.array(Image.open('./data/manual_label/' + row[0] + ".jpg")) for row in csv.reader(inputcsv) if len(row[1]) == 6] # 只讀答案是6位的
+inputcsv = open('./data/manual_label/captcha_vali.csv', 'r', encoding = 'utf8')
+oldanswer = [row[1] for row in csv.reader(inputcsv) if len(row[1]) == 6] # 只讀答案是6位的
+answer = []
+datagen = ImageDataGenerator(rotation_range=5,shear_range=0.2,zoom_range=0.05,fill_mode='nearest')
+index, augmentindex, oldanswerindex = 0, 0, 0
+for img in data:
+    for batch in datagen.flow(np.asarray([img]), batch_size=1):
+        index += 1
+        augmentindex += 1
+        batch = batch.reshape((60,200,3))
+        Image.fromarray(np.uint8(batch)).convert("RGB").save("./data/6_real_train_set/" + str(index) + ".jpg", "JPEG")
+        answer.append((str(index), oldanswer[oldanswerindex]))
+        if augmentindex >= 50: # 每張產生50個
+            oldanswerindex += 1
+            augmentindex = 0
+            break
+csv.writer(outputcsv).writerows(answer)

data_augment.py

@@ -1,14 +1,87 @@
 from keras.models import load_model
 from keras.models import Model
-from PIL import Image, ImageEnhance
+from keras import backend as K
+from PIL import Image
 import numpy as np
+import os
+import csv
+LETTERSTR = "0123456789ABCDEFGHJKLMNPQRSTUVWXYZ"
 
-validate_data = np.stack([(np.array(Image.open("./data/8dataset/8-" + str(index) +".jpg")))/255.0 for index in range(1, 11, 1)])
-model = load_model("./data/cnn_model.h5")
-prediction = model.predict(validate_data)
-resultlist = ["" for _ in range(10)]
-for predict in prediction:
-    for index in range(10):
-        resultlist[index] += str(np.argmax(predict[index]))
-for result in resultlist:
-    print(result)
+
+def toonehot(text):
+    labellist = []
+    for letter in text:
+        onehot = [0 for _ in range(34)]
+        num = LETTERSTR.find(letter)
+        onehot[num] = 1
+        labellist.append(onehot)
+    return labellist
+
+
+print("Loading test data...")
+testcsv = open('./data/manual_label/captcha_test.csv', 'r', encoding = 'utf8')
+test_data = np.stack([np.array(Image.open("./data/manual_label/" + row[0] + ".jpg"))/255.0 for row in csv.reader(testcsv)])
+testcsv = open('./data/manual_label/captcha_test.csv', 'r', encoding = 'utf8')
+test_label = [row[1] for row in csv.reader(testcsv)]
+print("Loading model...")
+K.clear_session()
+model = None
+model5 = load_model("./data/model/imitate_5_model.h5")
+model6 = load_model("./data/model/imitate_6_model.h5")
+model56 = load_model("./data/model/real_56_model.h5")
+print("Predicting...")
+prediction56 = [6 if arr[0] > 0.5 else 5 for arr in model56.predict(test_data)] # 5/6碼分類
+prediction5 = model5.predict(test_data) # 5碼
+prediction6 = model6.predict(test_data) # 6碼
+
+# 以下計算各個模型各個字元辨識率等等，有點亂，以後有空再整理
+total, total5, total6 = len(prediction56), 0, 0
+correct5, correct6, correct56, correct = 0, 0, 0, 0
+correct5digit, correct6digit = [0 for _ in range(5)], [0 for _ in range(6)]
+totalalpha, correctalpha = len([1 for ans in test_label for char in ans if char.isalpha()]), 0
+for i in range(total):
+    checkcorrect = True
+    if prediction56[i] == len(test_label[i]):
+        correct56 += 1
+    else:
+        checkcorrect = False
+    if prediction56[i] == 5:
+        total5 += 1
+        allequal = True
+        for char in range(5):
+            if LETTERSTR[np.argmax(prediction5[char][i])] == test_label[i][char]:
+                correct5digit[char] += 1
+                correctalpha += 1 if LETTERSTR[np.argmax(prediction5[char][i])].isalpha() else 0
+            else:
+                allequal = False
+        if allequal:
+            correct5 += 1
+        else:
+            checkcorrect = False
+    else:
+        total6 += 1
+        allequal = True
+        for char in range(6):
+            if LETTERSTR[np.argmax(prediction6[char][i])] == test_label[i][char]:
+                correct6digit[char] += 1
+                correctalpha += 1 if LETTERSTR[np.argmax(prediction6[char][i])].isalpha() else 0
+            else:
+                allequal = False
+        if allequal:
+            correct6 += 1
+        else:
+            checkcorrect = False
+    if checkcorrect:
+        correct += 1
+
+print("5 or 6 model acc:{:.4f}%".format(correct56/total*100)) # 5/6模型acc
+print("---------------------------")
+print("5digits model acc:{:.4f}%".format(correct5/total5*100)) # 5模型acc
+for i in range(5):
+    print("digit{:d} acc:{:.4f}%".format(i+1, correct5digit[i]/total5*100)) # 5模型各字元acc
+print("---------------------------")
+print("6digits model acc:{:.4f}%".format(correct6/total6*100)) # 6模型acc
+for i in range(6):
+    print("digit{:d} acc:{:.4f}%".format(i+1, correct6digit[i]/total6*100)) # 6模型各字元acc
+print("---------------------------")
+print("alpha acc:{:.4f}%".format(correctalpha/totalalpha*100)) # 整體英文字acc

demo_cnn.py

@@ -0,0 +1,50 @@
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import numpy as np
+from PIL import Image
+from keras.models import load_model, Model
+import time
+import random
+IDNumber = "X123456789" # 填入你的身分證字號
+model = None
+model5 = load_model("./data/model/imitate_5_model.h5") # 辨識5碼的Model
+model6 = load_model("./data/model/imitate_6_model.h5") # 辨識6碼的Model
+model56 = load_model("./data/model/real_56_model.h5") # 辨識是5碼or6碼的Model
+LETTERSTR = "0123456789ABCDEFGHJKLMNPQRSTUVWXYZ"
+driver = webdriver.Chrome("./data/chromedriver.exe") # chromedriver 路徑
+correct, wrong = 0, 0
+
+for _ in range(1000):# 跑1000次
+    driver.get('http://railway1.hinet.net/Foreign/TW/ecsearch.html')
+    id_textbox = driver.find_element_by_id('person_id')
+    id_textbox.send_keys(IDNumber)
+    button = driver.find_element_by_css_selector('body > div.container > div.row.contents > div > form > div > div.col-xs-12 > button')
+    button.click()
+    driver.save_screenshot('tmp.png')
+    location = driver.find_element_by_id('idRandomPic').location
+    x, y = location['x'] + 5, location['y'] + 5
+    img = Image.open('tmp.png')
+    captcha = img.crop((x, y, x+200, y+60))
+    captcha.convert("RGB").save('captcha.jpg', 'JPEG')
+    # check is 5 or 6 digits
+    p56 = model56.predict(np.stack([np.array(Image.open('captcha.jpg'))/255.0]))[0][0]
+    if p56 > 0.5:
+        model = model6
+    else:
+        model = model5
+    prediction = model.predict(np.stack([np.array(Image.open('captcha.jpg'))/255.0]))
+    answer = ""
+    for predict in prediction:
+        answer += LETTERSTR[np.argmax(predict[0])]
+    captcha_textbox = driver.find_element_by_id('randInput')
+    captcha_textbox.send_keys(answer)
+    driver.find_element_by_id('sbutton').click()
+    if "亂數號碼錯誤" in driver.page_source:
+        wrong += 1
+    else:
+        correct += 1
+    print("{:.4f}% (Correct{:d}-Wrong{:d})".format(correct/(correct+wrong)*100, correct, wrong))
+    time.sleep(3)