example/cola_lab_tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## Metric-Visualizer Cola-lab Tutorial"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "!pip install -e ..\n",
    "# !pip install metric_visualizer"
   ],
   "metadata": {
    "collapsed": false
   },
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Metrics Logging\n",
    "This example shows how to log metrics into metric-visualizer"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "import os\n",
    "import random\n",
    "\n",
    "import findfile\n",
    "\n",
    "from metric_visualizer import MetricVisualizer\n",
    "import numpy as np\n",
    "\n",
    "MV = MetricVisualizer(name=\"example\", trial_tag=\"Model\")\n",
    "\n",
    "# 随机指标模拟\n",
    "repeat = 100  # number of repeats of each trial i.e., 每个实验模型/算法/目标/etc的重复次数\n",
    "metric_num = 2  # number of metrics in each trial  i.e., 每个模型/算法/目标/etc的指标总数\n",
    "\n",
    "trial_names = [\"NSGA-II\", \"NSGA-III\", \"MOEA/D\"]  # fake trial names\n",
    "\n",
    "for r in range(repeat):  # repeat the experiments to plot violin or box figure\n",
    "    metrics = [\n",
    "        (np.random.random() + n + (1 if random.random() > 0.5 else -1))\n",
    "        for n in range(metric_num)\n",
    "    ]  # n is metric scale factor\n",
    "    for n_trial in range(len(trial_names)):\n",
    "        for i, m in enumerate(metrics):\n",
    "            MV.log_metric(\n",
    "                trial_name=\"Trial-{}\".format(n_trial),\n",
    "                metric_name=\"metric{}\".format(i + 1),\n",
    "                value=m * random.random(),\n",
    "            )  # add metric by custom name and value"
   ],
   "metadata": {
    "collapsed": false
   },
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Metrics Summary\n",
    "This example shows how to view simple metrics statistics"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "# MV.remove_outliers()  # remove outliers for each metric, do not use this unless you have to\n",
    "\n",
    "MV.summary(save_path=\"./example.summary.txt\", no_print=False)\n",
    "\n",
    "MV.to_execl(save_path=\"example.xlsx\")  # save summary to excel\n",
    "# other usages\n",
    "# MV.to_csv()\n",
    "# MV.to_txt()\n",
    "# MV.to_json()\n",
    "# MV.to_latex()"
   ],
   "metadata": {
    "collapsed": false
   },
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Metrics Visualization\n",
    "This example shows how to visualize and compare the performance between different trials"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Box Plot by Tikz"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "fig_path = MV.box_plot(\n",
    "    engine=\"tikz\",\n",
    "    save_path=\"box_plot_by_trial.tex\",\n",
    ")"
   ],
   "metadata": {
    "collapsed": false
   },
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Important: Reformat to Cola-lab Template\n",
    "**Tip: Use default matplotlib setting and fine-tune the tikz styles will be better**\n",
    "Step1: Find a tikz plot template\n",
    "Step2: Get a tex plot file to reformat\n",
    "Step3: Fine-tune the plot tikz style settings\n",
    "Step4: Reformat\n",
    "Here is the full example:"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "from metric_visualizer import reformat_tikz_format_for_colalab\n",
    "\n",
    "# Step1: Let us use the template ./delta_hv.tex (Please find a suitable template based on your data)\n",
    "tex_template_path = \"./delta_hv.tex\"\n",
    "\n",
    "# Step2: Get a tex plot file\n",
    "fig_path = MV.box_plot(\n",
    "    engine=\"tikz\",\n",
    "    save_path=\"box_plot_by_trial.tex\",\n",
    "    no_overlap=True,  # this will separate different metrics\n",
    ")\n",
    "\n",
    "# Step3: Fine-tune the plot tikz style settings, these style settings will be written into tikz source code. So only tikz grammar is allowed\n",
    "# e.g.,\n",
    "style_settings = {\n",
    "    \"xtick\": {0, 1, 2},  # to be written as xtick = {0,1,2} in .tex file\n",
    "    \"font\": r\"\\LARGE\",  # to be written as font=\\LARGE in .tex file\n",
    "    \"legend pos\": \"south east\",  # to be written as legend pos=south east in .tex file\n",
    "    \"line width\": \"2pt\",\n",
    "    \"xlabel\": \"Algorithm\",\n",
    "    \"ylabel\": \"HV\",\n",
    "}\n",
    "\n",
    "# Step4: Reformat\n",
    "pdf_path = reformat_tikz_format_for_colalab(\n",
    "    template=tex_template_path,  # can be a file path or text source texts\n",
    "    tex_src_to_format=fig_path,  # can be a file path or text source texts\n",
    "    output_path=fig_path,  # can be a file path\n",
    "    style_settings=style_settings,\n",
    ")\n",
    "os.system('xdg-open \"{}\"'.format(pdf_path))  # for linux\n",
    "os.system(r'explorer \"file:{}\"'.format(pdf_path))  # for windows"
   ],
   "metadata": {
    "collapsed": false
   },
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Box Plot by Matplotlib\n",
    "You need to install latex and compile the tex source file to get the tikz plots"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "tikz_file_path = MV.box_plot(\n",
    "    show=True,\n",
    "    save_path=\"box_plot_by_trial.png\",\n",
    ")\n",
    "MV.compile_tikz(crop=True, clean=True)"
   ],
   "metadata": {
    "collapsed": false
   },
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### You can also plot by metrics\n",
    "All features are available for metric-based visualizations"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "fig_path = MV.box_plot(\n",
    "    by=\"metric\",\n",
    "    show=True,\n",
    "    save_path=\"box_plot_by_trial.png\",\n",
    ")"
   ],
   "metadata": {
    "collapsed": false
   },
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Other Available Plots"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "MV.box_plot(by=\"trial\", show=True)\n",
    "MV.violin_plot(by=\"trial\", show=True)\n",
    "MV.bar_plot(by=\"trial\", show=True)\n",
    "MV.bar_plot(by=\"trial\", show=True)\n",
    "MV.trajectory_plot(by=\"trial\", show=True)\n",
    "MV.sk_rank_plot(plot_type=\"box\", show=True)\n",
    "MV.sk_rank_plot(plot_type=\"violin\", show=True)\n",
    "MV.a12_bar_plot(show=True)\n",
    "\n",
    "\n",
    "MV.summary(dump_path=os.getcwd())\n",
    "MV.box_plot(\n",
    "    by=\"trial\",\n",
    "    engine=\"tikz\",\n",
    "    show=True,\n",
    "    save_path=\"box_plot_by_trial.tex\",\n",
    ")\n",
    "MV.violin_plot(\n",
    "    by=\"trial\",\n",
    "    engine=\"tikz\",\n",
    "    show=True,\n",
    "    save_path=\"violin_plot_by_trial.tex\",\n",
    ")\n",
    "MV.bar_plot(\n",
    "    by=\"trial\",\n",
    "    engine=\"tikz\",\n",
    "    show=True,\n",
    "    save_path=\"bar_plot_by_trial.tex\",\n",
    ")\n",
    "MV.bar_plot(\n",
    "    by=\"trial\",\n",
    "    engine=\"tikz\",\n",
    "    show=True,\n",
    "    save_path=\"bar_plot_by_trial.tex\",\n",
    ")\n",
    "MV.trajectory_plot(\n",
    "    by=\"trial\",\n",
    "    engine=\"tikz\",\n",
    "    show=True,\n",
    "    save_path=\"trajectory_plot_by_trial.tex\",\n",
    ")\n",
    "MV.sk_rank_plot(\n",
    "    plot_type=\"box\",\n",
    "    engine=\"tikz\",\n",
    "    show=True,\n",
    "    save_path=\"sk_rank_plot_box.tex\",\n",
    ")\n",
    "MV.sk_rank_plot(\n",
    "    plot_type=\"violin\",\n",
    "    engine=\"tikz\",\n",
    "    show=True,\n",
    "    save_path=\"sk_rank_plot_violin.tex\",\n",
    ")\n",
    "MV.a12_bar_plot(\n",
    "    plot_type=\"violin\",\n",
    "    engine=\"tikz\",\n",
    "    show=True,\n",
    "    save_path=\"sk_rank_plot_violin.tex\",\n",
    ")"
   ],
   "metadata": {
    "collapsed": false
   },
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Wilcoxon Rank Test (Mann Whitney U test)\n",
    "This example shows how to get the rank test results"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Rank by Trial"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "source": [
    "rank_test_results = MV.rank_test_by_trail(\n",
    "    trial=\"Trial-1\", rank_type=\"two-sided\"\n",
    ")  # rank_type = {'two-sided', 'less', 'greater'}, optional. i.e., the alternative param in scipy.stats.ranksum\n",
    "print(rank_test_results)"
   ],
   "metadata": {
    "collapsed": false
   },
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "source": [
    "rank_test_results = MV._rank_test_by_trial(\n",
    "    rank_type=\"two-sided\"\n",
    ")  # his will give you all rank test results\n",
    "print(rank_test_results)"
   ],
   "metadata": {
    "collapsed": false
   },
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Rank by Metric"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "source": [
    "rank_test_results = MV.rank_test_by_metric(\n",
    "    rank_type=\"greater\"\n",
    ")  # rank_type = {'two-sided', 'less', 'greater'}, optional. i.e., the alternative param in scipy.stats.ranksum\n",
    "print(rank_test_results)\n",
    "# rank_test_results = MV._rank_test_by_metric(rank_type=\"greater\")"
   ],
   "metadata": {
    "collapsed": false
   },
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Load Metric-Visualizer File Anywhere\n",
    "You can load a \".mv\" file or a list of \".mv\" files to merge example results, this is useful when you have to run experiments iteratively."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "source": [
    "mvs = findfile.find_cwd_files(\n",
    "    [\".mv\"], exclude_key=[\".pdf\", \"png\", \".aux\", \".log\", \".txt\"]\n",
    ")\n",
    "loaded_mv = MetricVisualizer.load(mvs)\n",
    "\n",
    "loaded_mv.summary()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%m'v\n"
    }
   },
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Load Metric-Visualizer File from Console or Bash\n",
    "\n",
    "```bash\n",
    "mvis *.mv\n",
    "```"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}