{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Final of Program Skripsi Multi Label Classification.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "_AeAly5PEJLX"
      },
      "source": [
        "# Import Library\n",
        "import shutil \n",
        "import tensorflow as tf\n",
        "import keras\n",
        "import time\n",
        "import math\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from datetime import datetime\n",
        "from google.colab import drive\n",
        "from tqdm import tqdm\n",
        "\n",
        "from keras.models import Model\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout, Flatten\n",
        "from keras.layers import Conv2D, MaxPooling2D\n",
        "from keras.layers import BatchNormalization\n",
        "from keras.preprocessing import image\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
        "from keras.wrappers.scikit_learn import KerasClassifier\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.model_selection import cross_val_score\n",
        "\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ywrq9_Ot4pTc"
      },
      "source": [
        "# timer \r\n",
        "class TimerError(Exception):\r\n",
        "  \"\"\"A custom exception used to report errors in use of Timer class\"\"\"\r\n",
        "class Timer:\r\n",
        "  def __init__(self):\r\n",
        "    self._start_time = None\r\n",
        "  def start(self):\r\n",
        "    \"\"\"Start a new timer\"\"\"\r\n",
        "    if self._start_time is not None:\r\n",
        "      raise TimerError(f\"Timer is running. Use .stop() to stop it\")\r\n",
        "\r\n",
        "    self._start_time = time.perf_counter()\r\n",
        "\r\n",
        "  def stop(self):\r\n",
        "    \"\"\"Stop the timer, and report the elapsed time\"\"\"\r\n",
        "    if self._start_time is None:\r\n",
        "      raise TimerError(f\"Timer is not running. Use .start() to start it\")\r\n",
        "\r\n",
        "    elapsed_time = time.perf_counter() - self._start_time\r\n",
        "    self._start_time = None\r\n",
        "    print(f\"Executed time: {elapsed_time:0.4f} seconds\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eXxc_cQN3As9"
      },
      "source": [
        "# load images and datset csv\n",
        "image_directory = 'gdrive/My Drive/Dataset Skripsi/DS4 - ALL/'\n",
        "df = pd.read_csv('gdrive/My Drive/Dataset Skripsi/DS4_CSV.csv')  \n",
        "\n",
        "test_directory = 'gdrive/My Drive/Dataset Skripsi/Dataset Testing - Resized - All/'  \n",
        "df_test = pd.read_csv('gdrive/My Drive/Dataset Skripsi/DATASET_TESTING.csv') "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vow7ZlDlEb7C"
      },
      "source": [
        "# preprocessing images\n",
        "SIZE = 200\n",
        "X_dataset = []  \n",
        "for i in tqdm(range(df.shape[0])):\n",
        "    img = image.load_img(image_directory +df['Name'][i]+'.jpg', target_size=(SIZE,SIZE,3))\n",
        "    img = image.img_to_array(img)\n",
        "    img = img/255.\n",
        "    X_dataset.append(img)\n",
        "    \n",
        "X = np.array(X_dataset)                     # array images\n",
        "y = np.array(df.drop(['Name'], axis=1))     # array output class label\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0, test_size=0.25)     # split for training and validation"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w08fVd_MsZjn"
      },
      "source": [
        "# define model cnn\r\n",
        "model = Sequential()\r\n",
        "\r\n",
        "model.add(Conv2D(filters=16, kernel_size=(5, 5), activation=\"relu\", input_shape=(SIZE,SIZE,3)))\r\n",
        "model.add(BatchNormalization())\r\n",
        "model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "model.add(Dropout(0.2))\r\n",
        "\r\n",
        "model.add(Conv2D(filters=32, kernel_size=(5, 5), activation='relu'))\r\n",
        "model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "model.add(BatchNormalization())\r\n",
        "model.add(Dropout(0.2))\r\n",
        "\r\n",
        "model.add(Conv2D(filters=64, kernel_size=(5, 5), activation=\"relu\"))\r\n",
        "model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "model.add(BatchNormalization())\r\n",
        "model.add(Dropout(0.2))\r\n",
        "\r\n",
        "model.add(Conv2D(filters=128, kernel_size=(5, 5), activation='relu'))\r\n",
        "model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "model.add(BatchNormalization())\r\n",
        "model.add(Dropout(0.2))\r\n",
        "\r\n",
        "model.add(Conv2D(filters=300, kernel_size=(5, 5), activation='relu'))\r\n",
        "model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "model.add(BatchNormalization())\r\n",
        "model.add(Dropout(0.2))\r\n",
        "\r\n",
        "model.add(Flatten())\r\n",
        "model.add(Dense(128, activation='relu'))\r\n",
        "model.add(Dropout(0.5))\r\n",
        "model.add(Dense(64, activation='relu'))\r\n",
        "model.add(Dropout(0.5))\r\n",
        "model.add(Dense(32, activation='relu'))\r\n",
        "model.add(Dropout(0.5))\r\n",
        "model.add(Dense(6, activation='sigmoid'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vThsHC1km-xs"
      },
      "source": [
        "# define model cnn 2\r\n",
        "weight = 1\r\n",
        "model = Sequential()\r\n",
        "\r\n",
        "model.add(Conv2D(filters=16, kernel_size=(5, 5), activation=\"relu\", input_shape=(SIZE,SIZE,3), kernel_initializer=keras.initializers.GlorotUniform(seed=1231)))\r\n",
        "model.add(BatchNormalization())\r\n",
        "model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "model.add(Dropout(0.2))\r\n",
        "\r\n",
        "model.add(Conv2D(filters=32, kernel_size=(5, 5), activation='relu',kernel_initializer=keras.initializers.GlorotUniform(seed=1231)))\r\n",
        "model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "model.add(BatchNormalization())\r\n",
        "model.add(Dropout(0.2))\r\n",
        "\r\n",
        "model.add(Conv2D(filters=64, kernel_size=(5, 5), activation=\"relu\",kernel_initializer=keras.initializers.GlorotUniform(seed=1231)))\r\n",
        "model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "model.add(BatchNormalization())\r\n",
        "model.add(Dropout(0.2))\r\n",
        "\r\n",
        "model.add(Conv2D(filters=128, kernel_size=(5, 5), activation='relu',kernel_initializer=keras.initializers.GlorotUniform(seed=1231)))\r\n",
        "model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "model.add(BatchNormalization())\r\n",
        "model.add(Dropout(0.2))\r\n",
        "\r\n",
        "model.add(Conv2D(filters=300, kernel_size=(5, 5), activation='relu',kernel_initializer=keras.initializers.GlorotUniform(seed=1231)))\r\n",
        "model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "model.add(BatchNormalization())\r\n",
        "model.add(Dropout(0.2))\r\n",
        "\r\n",
        "model.add(Flatten())\r\n",
        "model.add(Dense(128, activation='relu',kernel_initializer=keras.initializers.GlorotUniform(seed=66)))\r\n",
        "model.add(Dropout(0.5))\r\n",
        "model.add(Dense(64, activation='relu',kernel_initializer=keras.initializers.GlorotUniform(seed=66)))\r\n",
        "model.add(Dropout(0.5))\r\n",
        "model.add(Dense(32, activation='relu',kernel_initializer=keras.initializers.GlorotUniform(seed=66)))\r\n",
        "model.add(Dropout(0.5))\r\n",
        "model.add(Dense(6, activation='sigmoid',kernel_initializer=keras.initializers.GlorotUniform(seed=66)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nE2OAKm-4MPR"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1_iJdLw4shYA"
      },
      "source": [
        "### step decay learning rate ###\r\n",
        "initial_learning_rate = 0.0001\r\n",
        "def lr_step_decay(epoch, lr):\r\n",
        "    drop_rate = 0.3\r\n",
        "    epochs_drop = 10.0\r\n",
        "    return initial_learning_rate * math.pow(drop_rate, math.floor(epoch/epochs_drop))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j3VCnNyNi0cP"
      },
      "source": [
        "### time base decay learning rate\r\n",
        "initial_learning_rate = 0.0001\r\n",
        "epochs = 100\r\n",
        "decay = initial_learning_rate / epochs\r\n",
        "def lr_time_based_decay(epoch, lr):\r\n",
        "    return lr * 1 / (1 + decay * epoch)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2kuiAUt0pqbb"
      },
      "source": [
        "### exponential decay learning rate \r\n",
        "initial_learning_rate = 0.01\r\n",
        "def lr_exp_decay(epoch, lr):\r\n",
        "    k = 0.1\r\n",
        "    return initial_learning_rate * math.exp(-k*epoch)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ADNbMqasvGk"
      },
      "source": [
        "pat = 5 #this is the number of epochs with no improvment after which the training will stop\r\n",
        "early_stopping = EarlyStopping(monitor='val_loss', patience=pat, verbose=1)\r\n",
        "\r\n",
        "#define the model checkpoint callback -> this will keep on saving the model as a physical file\r\n",
        "model_checkpoint = ModelCheckpoint('model_16_2.h5', verbose=1, save_best_only=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dhe9Bw5F-yPZ"
      },
      "source": [
        "# laod tensorboard\r\n",
        "%load_ext tensorboard\r\n",
        "logdir = \"logs/scalars/\" + datetime.now().strftime(\"%Y%m%d-%H%M%S\")\r\n",
        "tensorboard_callback = keras.callbacks.TensorBoard(log_dir=logdir)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fwD6kE9Nsywo",
        "cellView": "code"
      },
      "source": [
        "#@title Default title text\n",
        "#model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), \n",
        "#              loss='binary_crossentropy', \n",
        "#             metrics=[tf.keras.metrics.BinaryAccuracy(name=\"binary_accuracy\" ,threshold=0.8)])\n",
        "\n",
        "#history = model.fit(X_train, y_train, epochs=50, validation_data=(X_test, y_test), batch_size=30, \n",
        "#                    callbacks=[model_checkpoint])\n",
        "\n",
        "#################### use early stoping, learning rate scheduler ##################################\n",
        "\n",
        "model.compile(optimizer='Adam', loss='binary_crossentropy', metrics=[tf.keras.metrics.BinaryAccuracy(name=\"binary_accuracy\" ,threshold=0.8)])\n",
        "\n",
        "#model.compile(optimizer='SGD', loss='binary_crossentropy', metrics=['accuracy'])\n",
        "\n",
        "t = Timer()\n",
        "t.start()\n",
        "#history = model.fit(X_train, y_train, epochs=100, validation_data=(X_test, y_test), batch_size=90, \n",
        "#                    callbacks=[keras.callbacks.LearningRateScheduler(lr_time_based_decay, verbose=1)])\n",
        "#history = model.fit(X_train, y_train, epochs=1, validation_data=(X_test, y_test), batch_size=100, callbacks=[early_stopping, model_checkpoint])\n",
        "history = model.fit(X_train, y_train, epochs=100, validation_data=(X_test, y_test), batch_size=90, callbacks=[model_checkpoint])\n",
        "t.stop()\n",
        "\n",
        "#history = model.fit(X_train, y_train, epochs=100, validation_data=(X_test, y_test), batch_size=90, callbacks=[model_checkpoint])\n",
        "\n",
        "#history = model.fit(X, y, epochs=100, batch_size=50)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GW7so4jhtB3c"
      },
      "source": [
        "# predict the testing dataset\r\n",
        "predictions = model1.predict(XTest, verbose=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-pIIahn3ITM9"
      },
      "source": [
        "# save prediction to csv\r\n",
        "prediction = pd.DataFrame(predictions).to_csv('prediction_val_100.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SsfNx9yNiuXR"
      },
      "source": [
        "# evaluate overall testing prediction \r\n",
        "model.evaluate(XTest, YTest, verbose=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h1HgsNeteazg"
      },
      "source": [
        "# save history training\r\n",
        "hist_df = pd.DataFrame(history.history) \r\n",
        "\r\n",
        "# or save to csv: \r\n",
        "hist_csv_file = 'hist_acc_final_2.csv'\r\n",
        "with open(hist_csv_file, mode='w') as f:\r\n",
        "    hist_df.to_csv(f)\r\n",
        "\r\n",
        "# copy to google drive\r\n",
        "shutil.copy('hist_acc_final_2.csv', \"/content/gdrive/MyDrive/1. Hasil Training CNN/Learning Rate Scheduler\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c009fNLWe6UG"
      },
      "source": [
        "# save model cnn\r\n",
        "model.save('model_acc_100.h5')\r\n",
        "\r\n",
        "# copy to google colab\r\n",
        "!cp model_acc_100.h5 '/content/gdrive/MyDrive/1. Hasil Training CNN/Learning Rate Scheduler'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S0jmX5xWC_l9"
      },
      "source": [
        "# load image from google drive\r\n",
        "input = image.load_img('/content/gdrive/MyDrive/Dataset Skripsi/Dataset Testing - Resized/18. Isi kurang, Tutup rusak, Label rusak/TEST_IK_TR_LR2.jpg', target_size=(SIZE,SIZE,3))\r\n",
        "\r\n",
        "# preprocessing image\r\n",
        "input = image.img_to_array(input)\r\n",
        "input = input/255.\r\n",
        "plt.imshow(input)\r\n",
        "input = np.expand_dims(input, axis=0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZO3Racv5MWo6"
      },
      "source": [
        "# extract feature learning layer\r\n",
        "layer_outputs = [layer.output for layer in model.layers]\r\n",
        "activation_model = Model(inputs=model.input, outputs=layer_outputs)\r\n",
        "activations = activation_model.predict(input)\r\n",
        " \r\n",
        "def display_activation(activations, col_size, row_size, act_index): \r\n",
        "    activation = activations[act_index]\r\n",
        "    activation_index=0\r\n",
        "    fig, ax = plt.subplots(row_size, col_size, figsize=(row_size*2.5,col_size*1.5))\r\n",
        "   # plt.figure(figsize = (20,20))\r\n",
        "    for row in range(0,row_size):\r\n",
        "        for col in range(0,col_size):\r\n",
        "            ax[row][col].imshow(activation[0, :, :, activation_index], aspect='equal')\r\n",
        "            activation_index += 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4O1eb47iNbiB"
      },
      "source": [
        "# visualized the feature learning layer\r\n",
        "display_activation(activations, 4, 4, 17)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a1ylNT9ecG-r"
      },
      "source": [
        "#plot the training and validation accuracy and loss at each epoch\r\n",
        "loss = history.history['loss']\r\n",
        "val_loss = history.history['val_loss']\r\n",
        "epochs = range(1, len(loss) + 1)\r\n",
        "plt.plot(epochs, loss, 'y', label='Training loss')\r\n",
        "plt.plot(epochs, val_loss, 'r', label='Validation loss')\r\n",
        "plt.title('Training and Validation loss')\r\n",
        "plt.xlabel('Epochs')\r\n",
        "plt.ylabel('Loss')\r\n",
        "plt.legend()\r\n",
        "plt.show()\r\n",
        "\r\n",
        "\r\n",
        "acc = history.history['binary_accuracy']\r\n",
        "val_acc = history.history['val_binary_accuracy']\r\n",
        "plt.plot(epochs, acc, 'y', label='Training acc')\r\n",
        "plt.plot(epochs, val_acc, 'r', label='Validation acc')\r\n",
        "plt.title('Training and Validation accuracy')\r\n",
        "plt.xlabel('Epochs')\r\n",
        "plt.ylabel('Accuracy')\r\n",
        "plt.legend()\r\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eAjZSkaQQEAU"
      },
      "source": [
        "# read csv from google colab\r\n",
        "plotting = pd.read_csv('/content/gdrive/MyDrive/1. Hasil Training CNN/Learning Rate Scheduler/final graph_csv.csv')\\"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hx4z8oxf3MtJ"
      },
      "source": [
        "#plot the training and validation accuracy and loss at each epoch\n",
        "loss = plotting['loss']\n",
        "val_loss = plotting['val_loss']\n",
        "epochs = range(1, len(loss) + 1)\n",
        "plt.figure(figsize=(13,8))\n",
        "plt.plot(epochs, loss, 'y', label='Training loss')\n",
        "plt.plot(epochs, val_loss, 'r', label='Validation loss')\n",
        "plt.title('Training and Validation loss')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.legend()\n",
        "plt.show()\n",
        "\n",
        "\n",
        "\n",
        "acc = plotting['binary_accuracy']\n",
        "val_acc = plotting['val_binary_accuracy']\n",
        "plt.figure(figsize=(13,8))\n",
        "plt.plot(epochs, acc, 'y', label='Training acc')\n",
        "plt.plot(epochs, val_acc, 'r', label='Validation acc')\n",
        "plt.title('Training and Validation accuracy')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.legend()\n",
        "plt.show()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nY2bPvcbHib-"
      },
      "source": [
        "# save history training\r\n",
        "hist_df = pd.DataFrame(history.history) \r\n",
        "\r\n",
        "# or save to csv: \r\n",
        "hist_csv_file = 'history_KFold_{}.csv'.format(i)\r\n",
        "with open(hist_csv_file, mode='w') as f:\r\n",
        "    hist_df.to_csv(f)\r\n",
        "\r\n",
        "# copy to drive\r\n",
        "shutil.copy('history_KFold_{}.csv'.format(i), '/content/gdrive/MyDrive/1. Hasil Training CNN')\r\n",
        "\r\n",
        "\r\n",
        "# copy to google colab\r\n",
        "#!cp 'history_KFold_{}.csv'.format(i) '/content/gdrive/MyDrive/1. Hasil Training CNN'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MYMWmAaf_pnk"
      },
      "source": [
        "import shutil \r\n",
        "\r\n",
        "shutil.copy('History_KFold_{}.csv'.format(i), '/content/gdrive/MyDrive/1. Hasil Training CNN')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pWfz0kYZWeKz"
      },
      "source": [
        "model.save('Model_KFold{}.h5'.format(i))\r\n",
        "\r\n",
        "shutil.copy('Model_KFold_{}.h5'.format(i), '/content/gdrive/MyDrive/1. Hasil Training CNN')\r\n",
        "\r\n",
        "# copy to google colab\r\n",
        "#!cp m '/content/gdrive/MyDrive/1. Hasil Training CNN'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yDNoEEP6IF0e"
      },
      "source": [
        "# open csv file\r\n",
        "constant = pd.read_csv('/content/gdrive/MyDrive/1. Hasil Training CNN/hist_const_0_001.csv')\r\n",
        "gstep = pd.read_csv('/content/gdrive/MyDrive/1. Hasil Training CNN/hist_sd_0_001.csv')\r\n",
        "gtime = pd.read_csv('/content/gdrive/MyDrive/1. Hasil Training CNN/hist_tb_0_001.csv')\r\n",
        "gext = pd.read_csv('/content/gdrive/MyDrive/1. Hasil Training CNN/hist_ed_0_001.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nvkY2cUWI1RR"
      },
      "source": [
        "acc1 = gstep['binary_accuracy']\r\n",
        "acc2 = gtime['binary_accuracy']\r\n",
        "acc3 = gext['binary_accuracy']\r\n",
        "acc4 = constant['binary_accuracy']\r\n",
        "\r\n",
        "#val_acc = df['val_binary_accuracy']\r\n",
        "plt.figure(figsize=(12,7))\r\n",
        "plt.plot(epochs, acc4, 'g', label='Constant')\r\n",
        "plt.plot(epochs, acc1, 'y', label='Step decay')\r\n",
        "plt.plot(epochs, acc2, 'r', label='Time based')\r\n",
        "plt.plot(epochs, acc3, 'b', label='Exponential decay')\r\n",
        "\r\n",
        "#plt.plot(epochs, val_acc, 'r', label='Validation acc')\r\n",
        "plt.title('Training accuracy')\r\n",
        "plt.xlabel('Epochs')\r\n",
        "plt.ylabel('Accuracy')\r\n",
        "\r\n",
        "plt.legend()\r\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UEJQGdW22OYO"
      },
      "source": [
        "const = pd.read_csv('/content/gdrive/MyDrive/1. Hasil Training CNN/Copy of hist_lr_const.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xNcJxi28iTWL"
      },
      "source": [
        "lr1 = gstep['lr']\r\n",
        "lr2 = gtime['lr']\r\n",
        "lr3 = gext['lr']\r\n",
        "lr4 = const['lr']\r\n",
        "\r\n",
        "epochs = range(1, len(loss) + 1)\r\n",
        "plt.figure(figsize=(12,7))\r\n",
        "plt.plot(epochs, lr4, 'g', label='Constant')\r\n",
        "plt.plot(epochs, lr1, 'y', label='Step decay')\r\n",
        "plt.plot(epochs, lr2, 'r', label='Time based')\r\n",
        "plt.plot(epochs, lr3, 'b', label='Exponential decay')\r\n",
        "\r\n",
        "plt.title('Learning rate')\r\n",
        "plt.xlabel('Epochs')\r\n",
        "plt.ylabel('Learning rate')\r\n",
        "plt.legend()\r\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8_NMzVTU6WF9"
      },
      "source": [
        "loss1 = gstep['loss']\r\n",
        "loss2 = gtime['loss']\r\n",
        "loss3 = gext['loss']\r\n",
        "loss4 = constant['loss']\r\n",
        "\r\n",
        "#val_acc = df['val_binary_accuracy']\r\n",
        "plt.figure(figsize=(12,7))\r\n",
        "plt.plot(epochs, loss4, 'g', label='Constant')\r\n",
        "plt.plot(epochs, loss1, 'y', label='Step decay')\r\n",
        "plt.plot(epochs, loss2, 'r', label='Time based')\r\n",
        "plt.plot(epochs, loss3, 'b', label='Exponential decay')\r\n",
        "\r\n",
        "#plt.plot(epochs, val_acc, 'r', label='Validation acc')\r\n",
        "plt.title('Training loss')\r\n",
        "plt.xlabel('Epochs')\r\n",
        "plt.ylabel('Accuracy')\r\n",
        "\r\n",
        "plt.legend()\r\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OfVHyNUR-5cn"
      },
      "source": [
        "%tensorboard --logdir logs/scalars"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "euTwaAxG-SsR"
      },
      "source": [
        "!nvidia-smi -L"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2zSqWgKq-fcr"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4JQWz-uS-pKH"
      },
      "source": [
        "!lscpu |grep 'Model name'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VfWUaGQGAqtK"
      },
      "source": [
        "!lscpu | grep 'Core(s) per socket:'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e-LiCMJlAX1f"
      },
      "source": [
        "!lscpu | grep 'Thread(s) per core'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dBqvBBFZ-uSp"
      },
      "source": [
        "!free -h --si | awk  '/Mem:/{print $2}'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dZE7SI8lAczm"
      },
      "source": [
        "!df -h / | awk '{print $4}'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HoNhOHkM-3Zw"
      },
      "source": [
        "!cat /proc/cpuinfo"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QDvmBd2BWj5d"
      },
      "source": [
        "seed = 7\r\n",
        "np.random.seed(seed)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NZcSyer9s-HN"
      },
      "source": [
        "# cross validation\r\n",
        "kfold = KFold(n_splits=10, shuffle=True, random_state=seed)\r\n",
        "cvscores = []\r\n",
        "for train, test in kfold.split(X, y):\r\n",
        "  model = Sequential()\r\n",
        "\r\n",
        "  model.add(Conv2D(filters=16, kernel_size=(5, 5), activation=\"relu\", input_shape=(SIZE,SIZE,3)))\r\n",
        "  model.add(BatchNormalization())\r\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "  model.add(Dropout(0.2))\r\n",
        "\r\n",
        "  model.add(Conv2D(filters=32, kernel_size=(5, 5), activation='relu'))\r\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "  model.add(BatchNormalization())\r\n",
        "  model.add(Dropout(0.2))\r\n",
        "\r\n",
        "  model.add(Conv2D(filters=64, kernel_size=(5, 5), activation=\"relu\"))\r\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "  model.add(BatchNormalization())\r\n",
        "  model.add(Dropout(0.2))\r\n",
        "\r\n",
        "  model.add(Conv2D(filters=128, kernel_size=(5, 5), activation='relu'))\r\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "  model.add(BatchNormalization())\r\n",
        "  model.add(Dropout(0.2))\r\n",
        "\r\n",
        "  model.add(Conv2D(filters=300, kernel_size=(5, 5), activation='relu'))\r\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\r\n",
        "  model.add(BatchNormalization())\r\n",
        "  model.add(Dropout(0.2))\r\n",
        "\r\n",
        "  model.add(Flatten())\r\n",
        "  model.add(Dense(128, activation='relu'))\r\n",
        "  model.add(Dropout(0.5))\r\n",
        "  model.add(Dense(64, activation='relu'))\r\n",
        "  model.add(Dropout(0.5))\r\n",
        "  model.add(Dense(32, activation='relu'))\r\n",
        "  model.add(Dropout(0.5))\r\n",
        "  model.add(Dense(6, activation='sigmoid'))\r\n",
        " \r\n",
        "  model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[tf.keras.metrics.BinaryAccuracy(name=\"binary_accuracy\" ,threshold=0.8)])\r\n",
        "\t# Fit the model\r\n",
        "  model.fit(X[train], y=[train], epochs=100, batch_size=50, verbose=1)\r\n",
        "\t# evaluate the model\r\n",
        "  scores = model.evaluate(X[test], y[test], verbose=1)\r\n",
        "  print(\"%s: %.2f%%\" % (model.metrics_names[1], scores[1]*100))\r\n",
        "  cvscores.append(scores[1] * 100)\r\n",
        "print(\"%.2f%% (+/- %.2f%%)\" % (numpy.mean(cvscores), numpy.std(cvscores)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_sSd9y7Lm0bR"
      },
      "source": [
        "# import h5 model cnn file\n",
        "from keras.models import load_model\n",
        "model = load_model('/content/gdrive/MyDrive/1. Hasil Training CNN/KFold/Model_KFold_5.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6sfDR-T2__wH"
      },
      "source": [
        "# for test individualy and only show the present class label\r\n",
        "img = image.load_img('IK_TR_LTA16.jpg', target_size=(SIZE,SIZE,3))\r\n",
        "\r\n",
        "img = image.img_to_array(img)\r\n",
        "img = img/255.\r\n",
        "plt.imshow(img)\r\n",
        "img = np.expand_dims(img, axis=0)\r\n",
        "\r\n",
        "classes = np.array(df.columns[1:])\r\n",
        "proba = model.predict(img)  \r\n",
        "\r\n",
        "sorted_categories = np.argsort(proba[0])[:-7:-1]  \r\n",
        "\r\n",
        "threshold = 0.8\r\n",
        "\r\n",
        "for i in range(6): \r\n",
        "  if proba[0][sorted_categories[i]] > threshold :\r\n",
        "    print(\"{}\".format(classes[sorted_categories[i]])+\" ({:.3})\".format(proba[0][sorted_categories[i]]))\r\n",
        "\r\n",
        "model.evaluate(XTest, YTest, verbose=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f5toBOQ5ENoR"
      },
      "source": [
        "# for test individually and show all class label\r\n",
        "img = image.load_img('IMG_20201230_213044.jpg', target_size=(SIZE,SIZE,3))\r\n",
        "\r\n",
        "img = image.img_to_array(img)\r\n",
        "img = img/255.\r\n",
        "plt.imshow(img)\r\n",
        "img = np.expand_dims(img, axis=0)\r\n",
        "\r\n",
        "classes = np.array(df.columns[1:])\r\n",
        "proba = model.predict(img)  \r\n",
        "\r\n",
        "sorted_categories = np.argsort(proba[0])[:-7:-1]  #Get class names for top 10 categories\r\n",
        "\r\n",
        "#Print classes and corresponding probabilities\r\n",
        "for i in range(6):\r\n",
        "    print(\"{}\".format(classes[sorted_categories[i]])+\" ({:.3})\".format(proba[0][sorted_categories[i]]))"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}