1_1_train_test_split.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/LouisVanLangendonck/UPC-AML-ArchitectureClassif/blob/main/train_test_split.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Train-test split for already scraped Architectural Style Images"
      ],
      "metadata": {
        "id": "NjwHTW2-N8kr"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Specifiy the Projects MAIN DIRECTORY (in which in the data/unzipped/all_data file, all images are stored"
      ],
      "metadata": {
        "id": "zrskSQHFOE1j"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B_rd0abnNBVJ"
      },
      "outputs": [],
      "source": [
        "project_dir = '/content/drive/MyDrive/AML'"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Creating the new substructure in seperate train and test files..."
      ],
      "metadata": {
        "id": "9c92YfW6OyfO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import random\n",
        "import shutil\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount = True)\n",
        "\n",
        "data_path = os.path.join(project_dir, '/data/unzipped')\n",
        "img_path = os.path.join(project_dir, '/data/unzipped/all_data') #Specify where images are stored\n",
        "styles = os.listdir(img_path)\n",
        "try:\n",
        "  os.mkdir(os.path.join(data_path, 'train')) #Make empty direct. called train\n",
        "except:\n",
        "  pass\n",
        "\n",
        "train = os.path.join(data_path, 'train') \n",
        "\n",
        "try: \n",
        "  os.mkdir(os.path.join(data_path, 'test')) #Make empty direct. called test\n",
        "except:\n",
        "  pass\n",
        "\n",
        "test = os.path.join(data_path, 'test') "
      ],
      "metadata": {
        "id": "LViAnxegNFwk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Splitting the data PER BUILDING (instead of per image) in 80% - 20%, stratified fashion. And copying it to the respective folder..."
      ],
      "metadata": {
        "id": "FfxUnt7iO2pw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "style_list = ['baroque', 'contemporary', 'gothic', 'modernism', 'neoclassicism', 'noucentisme', 'renaissance', 'romanesque']\n",
        "\n",
        "for style in style_list:\n",
        "  try:\n",
        "    os.mkdir(os.path.join(train, style)) #Make an empty folder per style in train\n",
        "    os.mkdir(os.path.join(test, style)) #Make an empty folder per style in test\n",
        "  except:\n",
        "    pass\n",
        "\n",
        "for style in styles:\n",
        "    if style in style_list:\n",
        "        direc = os.path.join(img_path, style)\n",
        "        images = os.listdir(direc)\n",
        "        unique = set()\n",
        "        for image in images:\n",
        "            if '.jpg' in image:\n",
        "                unique.add(image.split('-')[1])  #Make list of unique building numbers (as there can be multiple images per building and we \n",
        "                #want these seperated between the splits)\n",
        "        test_ids = random.sample(unique, int(0.2*len(unique)))  #Take sample of 20% of UNIQUE BUILDINGS\n",
        "        print(style + ':')\n",
        "        print('There are {} distinct buildings in the test set'.format(len(test_ids)))\n",
        "        for image in images:\n",
        "            if '.jpg' in image:\n",
        "                src = os.path.join(direc, image)\n",
        "                if image.split('-')[1] in test_ids:\n",
        "                    target = os.path.join(test, style)   #copy corresponding image to test set\n",
        "                    shutil.copy(src, target)\n",
        "                else:\n",
        "                    target = os.path.join(train, style)  #Or if not to train set\n",
        "                    shutil.copy(src, target)\n",
        "        print('There are {} images in the train set'.format(len(os.listdir(os.path.join(train, style)))))\n",
        "        print('There are {} images in the test set'.format(len(os.listdir(os.path.join(test, style)))))"
      ],
      "metadata": {
        "id": "okDiSRsEN0pB"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}