proj_1_cv.py

# -*- coding: utf-8 -*-
"""Proj#1-CV.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1qCUW5rOrcw27liqwla1YaSKlBLw1Fa-V

# **Imports**
"""

#1
#Check CPU and RAM specifications
!cat /proc/cpuinfo
!cat /proc/meminfo

#2
#Check GPU specifications
from tensorflow.python.client import device_lib
device_lib.list_local_devices()

#3
!pip list

#4
import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np 
import torchvision
from torchvision import datasets, models, transforms
import matplotlib.pyplot as plt
import time
from google.colab import files, auth, drive
import os
import glob
from shutil import copyfile
import cv2 as cv
from google.colab.patches import cv2_imshow
import math
import torch.nn.functional as F

#5
CUDA_LAUNCH_BLOCKING=1

plt.ion()  

use_gpu = torch.cuda.is_available()
if use_gpu:
    print("Using CUDA")
else:
    print("Using CPU")

#6
# Mount
drive.mount('/content/gdrive', force_remount=True)
data_dir_drive ='/content/gdrive/My Drive/Colab Notebooks/CV/Proj1' #change to the project folder on the drive
data_dir ='/content' # When read from unziped file

#7
# Unzip dataset to /content
start = time.time()

!unzip -q '/content/gdrive/My Drive/Colab Notebooks/CV/Proj1/data/Project1.zip' -d '/content/gdrive/My Drive/Colab Notebooks/CV/Proj1/data'

print('Took', (time.time() - start), ' secundes to unzip')

#8
!ls '/content/gdrive/My Drive/Colab Notebooks/CV/Proj1/data'

#9
folder_in = data_dir_drive + '/data/images/'
folder_scanned_out = data_dir_drive + '/data/additional_data/1.scanned/'
folder_rotated_and_perspective_out = data_dir_drive + '/data/additional_data/2.rotated+perspective/'
folder_no_annotation_out = data_dir_drive + '/data/additional_data/3.no_annotation/'
folder_handwritten_out = data_dir_drive + '/data/additional_data/4.handwritten/'
folder_digits = data_dir_drive + '/data/digits/'
folder_models = data_dir_drive + '/models/'
folder_barem = data_dir_drive + '/data/ground-truth-correct-answers/'

input_folder = data_dir_drive + '/input_data/'

input_folder_scanned = input_folder + '1.scanned/'
input_folder_rotated_and_perspective = input_folder + '2.rotated+perspective/'
input_folder_no_annotation = input_folder + '3.no_annotation/'
input_folder_handwritten = input_folder + '4.handwritten/'

output_folder = data_dir_drive + '/output_data/'

#10
def make_additional_data_scanned():
    if not os.path.exists(folder_scanned_out):
        os.makedirs(folder_scanned_out)

    for f in glob.iglob(folder_scanned_out+ '/*'):
        os.remove(f)

    for i in range(1, 151):
        file = open(folder_in + 'image_' + str(i) +'.txt', 'r')
        first_line = file.readline()
        file.close() 
        first_line = first_line.replace('\n', '')
        info = first_line.split(' ')
        copyfile(folder_in + 'image_' + str(i) +'.jpg', folder_scanned_out + str(i) + '_scanned_' + info[0] + info[1] +'.jpg')

#11
def make_additional_data_rotated_and_perspective():
    if not os.path.exists(folder_rotated_and_perspective_out):
        os.makedirs(folder_rotated_and_perspective_out)

    for f in glob.iglob(folder_rotated_and_perspective_out + '/*'):
        os.remove(f)

    for i in range(1, 151):
        file = open(folder_in + 'image_' + str(i) +'.txt', 'r')
        first_line = file.readline()
        file.close() 
        first_line = first_line.replace('\n', '')
        info = first_line.split(' ')
        copyfile(folder_in + 'perspective_' + str(i) +'.jpg', folder_rotated_and_perspective_out + str(i) + '_perspective_' + info[0] + info[1] +'.jpg')

    for i in range(1, 151):
        file = open(folder_in + 'image_' + str(i) +'.txt', 'r')
        first_line = file.readline()
        file.close() 
        first_line = first_line.replace('\n', '')
        info = first_line.split(' ')
        copyfile(folder_in + 'rotation_' + str(i) +'.jpg', folder_rotated_and_perspective_out + str(i) + '_rotated_' + info[0] + info[1] +'.jpg')

#12
def make_additional_data_no_annotation():
    if not os.path.exists(folder_no_annotation_out):
        os.makedirs(folder_no_annotation_out)

    for f in glob.iglob(folder_no_annotation_out + '/*'):
        os.remove(f)

    for i in range(1, 151):
        copyfile(folder_in + 'image_' + str(i) +'.jpg', folder_no_annotation_out + str(i) + '_scanned.jpg')
        copyfile(folder_in + 'perspective_' + str(i) +'.jpg', folder_no_annotation_out + str(i) + '_perspective.jpg')
        copyfile(folder_in + 'rotation_' + str(i) +'.jpg', folder_no_annotation_out + str(i) + '_rotated.jpg')

#13
def make_additional_handwritten():
    pass

#14
def make_additional_data():
    make_additional_data_scanned()
    make_additional_data_rotated_and_perspective()
    make_additional_data_no_annotation()
    make_additional_handwritten()

#make_additional_data()

#15
SHOW_INTERMEDIATE_RESULTS = True
char_to_index = {'A': 0, 'B': 1, 'C': 2, 'D': 3}
index_to_char = {0: 'A', 1: 'B', 2: 'C', 3: 'D'}

#16
def print_img(image):
    if SHOW_INTERMEDIATE_RESULTS:
        cv2_imshow(cv.resize(image, (0, 0), fx=0.15, fy=0.15))

#17
ground_truth_hw = np.loadtxt(data_dir_drive + '/data/hw_data.txt', dtype=str)
ground_truth_hw_dict = {}
for [i, nr] in ground_truth_hw:
    ground_truth_hw_dict[int(i)] = nr
print(ground_truth_hw_dict)

"""# **Scenario 1 (real world)**
You receive a test set containing 55 scanned images annotated with the option (F or I) and with the digit (1, 2, 3 or 4). For each image you have
to output the corresponding grade.
"""

#1
def find_rows(grayscale_image):
    # find the edges on the Y axis
    # apply Sobel filter on the Y axis
    edges_y = cv.Sobel(grayscale_image, ddepth = cv.CV_64F, dx = 0, dy = 1)
    # normalize the output
    edges_y = np.abs(edges_y)
    edges_y = edges_y / edges_y.max()
            
    # convert to a black and white image
    _, edges_y_th = cv.threshold(edges_y, 0.4, 255, cv.THRESH_BINARY_INV) # the second param is the threshold
    #print_img(edges_y_th)
        
    # create a mask for black pixels
    mask = (edges_y_th == 0) * 1
    # find the rows from mask
    all_lines = np.sum(mask, axis = 1)
    all_lines = all_lines.argsort()

    num_lines = min(len(all_lines), 150)
    edges_y_th = np.dstack((edges_y_th, edges_y_th, edges_y_th))
    lines = [] #  _ x 
    for i in range(num_lines):
        cv.line(edges_y_th, (0, all_lines[-i]), (grayscale_image.shape[1], all_lines[-i]), (0, 0, 255), 2) 
        lines.append([(0, all_lines[-i]), (grayscale_image.shape[1], all_lines[-i])])
        
    #print_img(edges_y_th)
    
    # sort only on y
    lines.sort(key=lambda coords: coords[0][1])
    
    threshold_same_line = 30
    distict_lines = []   
    distict_lines.append(lines[0])
     
    for line in lines:  
        if line[0][1] - distict_lines[-1][0][1] > threshold_same_line:
            distict_lines.append(line)   
    
    # take the last 16 lines
    correct_lines = distict_lines[-16:]
    color_image = np.dstack((grayscale_image, grayscale_image, grayscale_image))
    for line in correct_lines: 
        cv.line(color_image, line[0], line[1], (255, 0, 0), 5) 
        
    #print_img(color_image)
    
    return correct_lines

#2
def find_columns(grayscale_image):
    # find the edges on the X axis
    edges_x = cv.Sobel(grayscale_image, ddepth = cv.CV_64F, dx = 1, dy = 0)
    # normalize the output 
    edges_x = np.abs(edges_x)
    edges_x = edges_x / edges_x.max()
        
    # convert to a black and white image 
    _, edges_x_th = cv.threshold(edges_x, 0.20, 255, cv.THRESH_BINARY_INV) # the second param is the threshold
    #print_img(edges_x_th)
        
    # create a mask for black pixels
    mask = (edges_x_th == 0) * 1
    all_cols = np.sum(mask, axis = 0)
    all_cols = all_cols.argsort()
    num_cols = 60
    edges_x_th = np.dstack((edges_x_th, edges_x_th, edges_x_th))
    cols = [] #  _ x 
    for i in range(num_cols):
        cv.line(edges_x_th, (all_cols[-i], 0), (all_cols[-i], grayscale_image.shape[0]), (0, 0, 255), 2)
        cols.append([(all_cols[-i], 0), (all_cols[-i], grayscale_image.shape[0])])   
        
    #print_img(edges_x_th)
    
    # sort only on x0
    cols.sort(key=lambda coords: coords[0][0])
    threshold_same_column = 30
    distinct_cols = []
    distinct_cols.append(cols[0])
    
    for col in cols:  
        if col[0][0] - distinct_cols[-1][0][0] > threshold_same_column:
            distinct_cols.append(col)
    
    # take the last 5 cols
    correct_cols = distinct_cols[-5:]
    color_image = np.dstack((grayscale_image, grayscale_image, grayscale_image))
    for col in correct_cols: 
        cv.line(color_image, col[0], col[1], (255, 0, 0), 5) 
    #print_img(color_image)
    
    return correct_cols

#3
def find_table(grayscale_image):
    cols = find_columns(grayscale_image.copy())
    rows = find_rows(grayscale_image.copy())
    x_min = cols[0][0][0]
    x_max = cols[-1][1][0]
    y_min = rows[0][0][1]
    y_max = rows[-1][1][1]
    
    table = grayscale_image[y_min:y_max, x_min:x_max] 
    image = np.dstack((grayscale_image, grayscale_image, grayscale_image))
    
    for i in range(min(5, len(cols))): 
        cv.line(image, cols[i][0], cols[i][1], (255, 0, 0), 5) 
    for i in range(min(16, len(rows))):  
        cv.line(image, rows[i][0], rows[i][1], (0, 0, 255), 5) 
    #print_img(image)
    #print_img(table)

    return table, [x_min, y_min, x_max, y_max], cols, rows

#4
# write a function that plots the patches containing X with green
# and patches containing blanks with red
# use here your classifier (test mode)

def find_x(grayscale_image, vertical_lines, horizontal_lines):
    # grayscale_image - input image containing the frame
    # vertical_lines - list with the vertical lines
    # horizontal_lines - list with horizontal lines
    # ground_truth - grounth truth content for a frame    
    mean_x = []
    mean_blank = [] 
    
    image = np.dstack((grayscale_image, grayscale_image, grayscale_image))
    x_color = (0, 255, 0)  # plot a patch containing an X with green color
    blank_color = (0, 0, 255)  # plot a patch containing a blank with red color  

    result = {}  
            
    # crop each patch and display it
    for i in range(len(horizontal_lines) - 1):
        mean_patches = []
        for j in range(len(vertical_lines) - 1):
            x_min = vertical_lines[j][0][0] + 15
            x_max = vertical_lines[j + 1][1][0] - 5
            y_min = horizontal_lines[i][0][1] + 15
            y_max = horizontal_lines[i + 1][1][1] - 5
            
            # write your code here
            patch = grayscale_image[y_min:y_max, x_min:x_max]

            mean_patch = np.round(patch.mean())
            mean_patches.append(mean_patch)

        x_pos_j = np.argmin(mean_patches)
        for j in range(len(vertical_lines) - 1):

            x_min = vertical_lines[j][0][0] + 15
            x_max = vertical_lines[j + 1][1][0] - 5
            y_min = horizontal_lines[i][0][1] + 15
            y_max = horizontal_lines[i + 1][1][1] - 5

            if j == x_pos_j:
                color = x_color
                if i + 1 not in result.keys():
                    result[i + 1] = [index_to_char[j]]
                else:
                    result[i + 1].append(index_to_char[j])
            else:
                color = blank_color
            
            cv.rectangle(image, (x_min, y_min), (x_max, y_max), color=color, thickness=5)
            #cv.putText(image, str(mean_patch)[:3] ,(x_min + 10, y_min + 50), cv.FONT_HERSHEY_COMPLEX, 1, (0,0,0), 2) 

    #print_img(image)
    return result, image

#5
# test 
acc = 0
for i in range(1, 151):
    image_name = 'image_' + str(i) + '.jpg'
   
    print(image_name)
    # load image

    image = cv.imread(folder_in + image_name)
    #print_img(image)

    # blur the image using a 7 by 7 kernel
    image = cv.GaussianBlur(image, (7, 7), 0)

    # take the lower part of the image
    orig_h, orig_w, _ = image.shape
    image = image[int(orig_h * 0.41):int(orig_h * 0.88)]

    # transform the image to grayscale
    grayscale_image = cv.cvtColor(image, cv.COLOR_BGR2GRAY)

    left_grayscale_image = grayscale_image[:, :orig_w//2]
    right_grayscale_image = grayscale_image[:, orig_w//2:]

    _, _, cols_l, rows_l = find_table(left_grayscale_image)
    _, _, cols_r, rows_r = find_table(right_grayscale_image)

    # load ground truth
    ground_truth_content = np.loadtxt(folder_in + 'image_' + str(i) + '.txt', dtype=str)
    ground_truth_left = ground_truth_content[1:16]
    ground_truth_right = ground_truth_content[16:-1]

    result_l, image_l = find_x(left_grayscale_image, cols_l, rows_l)
    #print(result_l)
    result_r, image_r = find_x(right_grayscale_image, cols_r, rows_r)
    #print(result_r)

    left_err = 0;
    for r in range(15):
        tr_ans = ground_truth_left[r][1]
        if r + 1 not in result_l.keys():
            print("LINIE NECOMPLETATA")
            left_err += 1
        else:
            if len(result_l[r+1]) > 1:
                print("PREA MULTE RANDURI COMPLETATE")
                left_err += 1
            else:
                if result_l[r+1][0] != tr_ans:
                    left_err = left_err + 1

    if left_err:
        print("GRESEALA PE STANGA")
        print_img(image_l)

    right_err = 0;
    for r in range(15):
        tr_ans = ground_truth_right[r][1]
        if r + 1 not in result_r.keys():
            print("LINIE NECOMPLETATA")
            right_err += 1
        else:
            if len(result_r[r+1]) > 1:
                print("PREA MULTE RANDURI COMPLETATE")
                right_err += 1
            else:
                if result_r[r+1][0] != tr_ans:
                    right_err = right_err + 1

    if right_err:
        print("GRESEALA PE DREAPTA")
        print_img(image_r)

    if left_err == 0 and right_err == 0:
        print("OK")

    acc = acc + right_err + left_err

#6
print('Total greseli = ' + str(acc))

#7
def predict_grade_scenario_1(path, subject, number):
    image = cv.imread(path)
    #print_img(image)

    # blur the image using a 7 by 7 kernel
    image = cv.GaussianBlur(image, (7, 7), 0)

    # take the lower part of the image
    orig_h, orig_w, _ = image.shape
    image = image[int(orig_h * 0.41):int(orig_h * 0.88)]

    # transform the image to grayscale
    grayscale_image = cv.cvtColor(image, cv.COLOR_BGR2GRAY)

    left_grayscale_image = grayscale_image[:, :orig_w//2]
    right_grayscale_image = grayscale_image[:, orig_w//2:]

    _, _, cols_l, rows_l = find_table(left_grayscale_image)
    _, _, cols_r, rows_r = find_table(right_grayscale_image)

    number_ok = 0
    # load ground truth correct answers / barem
    if subject == 'F':
        if number == 1:
            ground_truth_content = np.loadtxt(folder_barem + 'Fizica_varianta1.txt', dtype=str)
        elif number == 2:
            ground_truth_content = np.loadtxt(folder_barem + 'Fizica_varianta2.txt', dtype=str)
        elif number == 3:
            ground_truth_content = np.loadtxt(folder_barem + 'Fizica_varianta3.txt', dtype=str)
        else:
            ground_truth_content = np.loadtxt(folder_barem + 'Fizica_varianta4.txt', dtype=str)
    else:
        if number == 1:
            ground_truth_content = np.loadtxt(folder_barem + 'Informatica_varianta1.txt', dtype=str)
        elif number == 2:
            ground_truth_content = np.loadtxt(folder_barem + 'Informatica_varianta2.txt', dtype=str)
        elif number == 3:
            ground_truth_content = np.loadtxt(folder_barem + 'Informatica_varianta3.txt', dtype=str)
        else:
            ground_truth_content = np.loadtxt(folder_barem + 'Informatica_varianta4.txt', dtype=str)

    ground_truth_left = ground_truth_content[1:16]
    ground_truth_right = ground_truth_content[16:-1]

    result_l, image_l = find_x(left_grayscale_image, cols_l, rows_l)
    #print(result_l)
    result_r, image_r = find_x(right_grayscale_image, cols_r, rows_r)
    #print(result_r)

    left_mis = 0;
    for r in range(15):
        tr_ans = ground_truth_left[r][1]
        if r + 1 not in result_l.keys():
            left_mis += 1
        else:
            if len(result_l[r+1]) > 1:
                left_mis += 1
            else:
                if result_l[r+1][0] != tr_ans:
                    left_mis += 1
                else:
                    number_ok += 1

    right_mis = 0;
    for r in range(15):
        tr_ans = ground_truth_right[r][1]
        if r + 1 not in result_r.keys():
            right_mis += 1
        else:
            if len(result_r[r+1]) > 1:
                right_mis += 1
            else:
                if result_r[r+1][0] != tr_ans:
                    right_mis += 1
                else:
                    number_ok += 1

    return (number_ok * 3) / 10 + 1

#8
acc = 0
for i in range(1, 151):
    image_name = 'image_' + str(i) + '.jpg'
   
    print(image_name)
    file = open(folder_in + 'image_' + str(i) +'.txt', 'r')
    first_line = file.readline()
    file.close() 
    first_line = first_line.replace('\n', '')
    info = first_line.split(' ')

    grade = predict_grade_scenario_1(folder_in + image_name, info[0], int(info[1]))

    if ground_truth_hw_dict[i] == str(grade):
        acc += 1
    else:
        print("GRESEALA")

print("Acc = " + str(acc * 100 / 150) + " %")

"""# **Scenario 2 (intermediate)**
You receive a test set containing 50 rotated and 50 perspective images annotated with the view (rotated or perspective), the option (F or I) and the digit (1, 2, 3 or 4). For each image you have to output the corresponding grade
"""

#1
def transform_rotaded_and_perspective_in_scanned(img_path, template_path = data_dir_drive + '/data/template.jpg'):
    img_template = cv.imread(template_path)

    (h, w) = img_template.shape[:2]
    width = 3500
    r = width / float(w)
    dim = (width, int(h * r))
    img_template = cv.resize(img_template, dim, interpolation = cv.INTER_AREA)

    img_query = cv.imread(img_path)
    img_template = cv.cvtColor(img_template, cv.COLOR_BGR2RGB)
    img_query = cv.cvtColor(img_query, cv.COLOR_BGR2RGB)

    # create ORB object
    orb = cv.ORB_create(nfeatures=15000)
    # get the keypoints and the corresponding descriptors
    kp_template, des_template = orb.detectAndCompute(img_template, None)

    kp_query, des_query = orb.detectAndCompute(img_query, None) 

    # drawing keypoints
    drawing_img1 = np.copy(img_template)
    cv.drawKeypoints(img_template, kp_template, drawing_img1, flags=cv.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)

    # drawing keypoints
    drawing_img2 = np.copy(img_query)
    cv.drawKeypoints(img_query, kp_query, drawing_img2, flags=cv.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)

    #plt.figure(figsize=(14, 14))
    #plt.subplot(1, 2, 1)
    #plt.imshow(drawing_img1)
    #plt.subplot(1, 2, 2)
    #plt.imshow(drawing_img2)
    #plt.show()

    # help here: https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_feature2d/py_matcher/py_matcher.html
    # https://docs.opencv.org/master/dc/dc3/tutorial_py_matcher.html

    # create BFMatcher object
    # matcher takes normType, which is set to cv2.NORM_L2 for SIFT and SURF, cv2.NORM_HAMMING for ORB, FAST and BRIEF
    bf = cv.BFMatcher(cv.NORM_HAMMING, crossCheck=True)

    # Match descriptors.
    matches = bf.match(des_query, des_template) # query_image, train_image
    # Sort them in the order of their distance.
    matches = sorted(matches, key = lambda x:x.distance)

    # Draw first 50 matches.
    #plt.figure(figsize=(14, 14))
    #img3 = cv.drawMatches(img_template, kp_template, img_query, kp_query, matches[:50], None, flags=cv.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS)
    #plt.imshow(img3)
    #plt.show()

    matches = matches[:int(0.5 * len(matches))]
    points_template = np.zeros((len(matches), 2))
    points_query = np.zeros((len(matches), 2))

    for i, match in enumerate(matches):
        points_template[i, :] = kp_template[match.trainIdx].pt
        points_query[i, :] = kp_query[match.queryIdx].pt

    M, _ = cv.findHomography(points_query, points_template, cv.RANSAC, ransacReprojThreshold = 8)

    # use homography to get the aligned image 
    height, width, _ = img_template.shape # the shape with respect to the template image
    aligned_image2 = cv.warpPerspective(img_query, M, (width, height), flags=cv.INTER_NEAREST)

    return aligned_image2

#2
def set_global_cols_and_rows_for_scenario_2():
    img_template = cv.imread(data_dir_drive + '/data/template.jpg')
    (h, w) = img_template.shape[:2]
    width = 3500
    r = width / float(w)
    dim = (width, int(h * r))
    img_template = cv.resize(img_template, dim, interpolation = cv.INTER_AREA)
    orig_h, orig_w, _ = img_template.shape
    img_template = img_template[int(orig_h * 0.17):int(orig_h * 0.83)]
    img_template = img_template[:, int(orig_w * 0.05):int(orig_w * 0.95)]

    #print_img(image)
    img_template = cv.GaussianBlur(img_template,(7,7),0)
    img_template = cv.cvtColor(img_template, cv.COLOR_BGR2GRAY)

    left_img_template = img_template[:, :orig_w//2]
    right_img_template = img_template[:, orig_w//2:]

    _, _, cols_l, rows_l = find_table(left_img_template)
    _, _, cols_r, rows_r = find_table(right_img_template)
    return cols_l, rows_l, cols_r, rows_r

cols_l, rows_l, cols_r, rows_r = set_global_cols_and_rows_for_scenario_2()

#3
# test
acc_perspective = 0

for i in range(1, 151):
    image_name = 'perspective_' + str(i) + '.jpg'
   
    print(image_name)
    # load image

    try:
        aligned_image2 = transform_rotaded_and_perspective_in_scanned(folder_in + image_name)
        #print_img(aligned_image2)

        # take the lower part of the image
        orig_h, orig_w, _ = aligned_image2.shape
        aligned_image2 = aligned_image2[int(orig_h * 0.17):int(orig_h * 0.83)]
        image = aligned_image2[:, int(orig_w * 0.05):int(orig_w * 0.95)]

        #print_img(image)
        img_blur = cv.GaussianBlur(image,(7,7),0)
        grayscale_image = cv.cvtColor(img_blur, cv.COLOR_BGR2GRAY)
        th = cv.adaptiveThreshold(grayscale_image,255,cv.ADAPTIVE_THRESH_GAUSSIAN_C, cv.THRESH_BINARY,11,2)

        left_grayscale_image = th[:, :orig_w//2]
        right_grayscale_image = th[:, orig_w//2:]

        # load ground truth
        ground_truth_content = np.loadtxt(folder_in + 'image_' + str(i) + '.txt', dtype=str)
        ground_truth_left = ground_truth_content[1:16]
        ground_truth_right = ground_truth_content[16:-1]

        result_l, image_l = find_x(left_grayscale_image, cols_l, rows_l)
        #print(result_l)
        result_r, image_r = find_x(right_grayscale_image, cols_r, rows_r)
        #print(result_r)

        left_err = 0;
        for r in range(15):
            tr_ans = ground_truth_left[r][1]
            if r + 1 not in result_l.keys():
                print("LINIE NECOMPLETATA")
                left_err += 1
            else:
                if len(result_l[r+1]) > 1:
                    print("PREA MULTE RANDURI COMPLETATE")
                    left_err += 1
                else:
                    if result_l[r+1][0] != tr_ans:
                        left_err = left_err + 1

        if left_err:
            print("GRESEALA PE STANGA")
            print_img(image_l)

        right_err = 0;
        for r in range(15):
            tr_ans = ground_truth_right[r][1]
            if r + 1 not in result_r.keys():
                print("LINIE NECOMPLETATA")
                right_err += 1
            else:
                if len(result_r[r+1]) > 1:
                    print("PREA MULTE RANDURI COMPLETATE")
                    right_err += 1
                else:
                    if result_r[r+1][0] != tr_ans:
                        right_err = right_err + 1

        if right_err:
            print("GRESEALA PE DREAPTA")
            print_img(image_r)

        if left_err == 0 and right_err == 0:
            print("OK")
            acc_perspective = acc_perspective + 1
        else:
            print_img(th)

    except Exception as e:
        print(e)
 
print('ACC = ' + str(acc_perspective))

#4
# test  classifier 
acc_rotation = 0

for i in range(1, 151):
    image_name = 'rotation_' + str(i) + '.jpg'
   
    print(image_name)
    # load image

    try:
        aligned_image2 = transform_rotaded_and_perspective_in_scanned(folder_in + image_name)
        #print_img(aligned_image2)

        # take the lower part of the image
        orig_h, orig_w, _ = aligned_image2.shape
        aligned_image2 = aligned_image2[int(orig_h * 0.17):int(orig_h * 0.83)]
        image = aligned_image2[:, int(orig_w * 0.05):int(orig_w * 0.95)]

        #print_img(image)
        img_blur = cv.GaussianBlur(image,(7,7),0)
        grayscale_image = cv.cvtColor(img_blur, cv.COLOR_BGR2GRAY)
        th = cv.adaptiveThreshold(grayscale_image,255,cv.ADAPTIVE_THRESH_GAUSSIAN_C, cv.THRESH_BINARY,11,2)

        left_grayscale_image = th[:, :orig_w//2]
        right_grayscale_image = th[:, orig_w//2:]

        # load ground truth
        ground_truth_content = np.loadtxt(folder_in + 'image_' + str(i) + '.txt', dtype=str)
        ground_truth_left = ground_truth_content[1:16]
        ground_truth_right = ground_truth_content[16:-1]

        result_l, image_l = find_x(left_grayscale_image, cols_l, rows_l)
        #print(result_l)
        result_r, image_r = find_x(right_grayscale_image, cols_r, rows_r)
        #print(result_r)

        left_err = 0;
        for r in range(15):
            tr_ans = ground_truth_left[r][1]
            if r + 1 not in result_l.keys():
                print("LINIE NECOMPLETATA")
                left_err += 1
            else:
                if len(result_l[r+1]) > 1:
                    print("PREA MULTE RANDURI COMPLETATE")
                    left_err += 1
                else:
                    if result_l[r+1][0] != tr_ans:
                        left_err = left_err + 1

        if left_err:
            print("GRESEALA PE STANGA")
            print_img(image_l)

        right_err = 0;
        for r in range(15):
            tr_ans = ground_truth_right[r][1]
            if r + 1 not in result_r.keys():
                print("LINIE NECOMPLETATA")
                right_err += 1
            else:
                if len(result_r[r+1]) > 1:
                    print("PREA MULTE RANDURI COMPLETATE")
                    right_err += 1
                else:
                    if result_r[r+1][0] != tr_ans:
                        right_err = right_err + 1

        if right_err:
            print("GRESEALA PE DREAPTA")
            print_img(image_r)

        if left_err == 0 and right_err == 0:
            print("OK")
            acc_rotation += 1
        else:
            print_img(th)
    except Exception as e:
        print(e)
 
print('ACC = ' + str(acc_rotation))

#5
def predict_grade_scenario_2(path, subject, number):
    cols_l, rows_l, cols_r, rows_r = set_global_cols_and_rows_for_scenario_2()

    aligned_image2 = transform_rotaded_and_perspective_in_scanned(path)
    #print_img(aligned_image2)

    # take the lower part of the image
    orig_h, orig_w, _ = aligned_image2.shape
    aligned_image2 = aligned_image2[int(orig_h * 0.17):int(orig_h * 0.83)]
    image = aligned_image2[:, int(orig_w * 0.05):int(orig_w * 0.95)]

    #print_img(image)
    img_blur = cv.GaussianBlur(image,(7,7),0)
    grayscale_image = cv.cvtColor(img_blur, cv.COLOR_BGR2GRAY)
    th = cv.adaptiveThreshold(grayscale_image,255,cv.ADAPTIVE_THRESH_GAUSSIAN_C, cv.THRESH_BINARY,11,2)

    left_grayscale_image = th[:, :orig_w//2]
    right_grayscale_image = th[:, orig_w//2:]

    number_ok = 0
    # load ground truth correct answers / barem
    if subject == 'F':
        if number == 1:
            ground_truth_content = np.loadtxt(folder_barem + 'Fizica_varianta1.txt', dtype=str)
        elif number == 2:
            ground_truth_content = np.loadtxt(folder_barem + 'Fizica_varianta2.txt', dtype=str)
        elif number == 3:
            ground_truth_content = np.loadtxt(folder_barem + 'Fizica_varianta3.txt', dtype=str)
        else:
            ground_truth_content = np.loadtxt(folder_barem + 'Fizica_varianta4.txt', dtype=str)
    else:
        if number == 1:
            ground_truth_content = np.loadtxt(folder_barem + 'Informatica_varianta1.txt', dtype=str)
        elif number == 2:
            ground_truth_content = np.loadtxt(folder_barem + 'Informatica_varianta2.txt', dtype=str)
        elif number == 3:
            ground_truth_content = np.loadtxt(folder_barem + 'Informatica_varianta3.txt', dtype=str)
        else:
            ground_truth_content = np.loadtxt(folder_barem + 'Informatica_varianta4.txt', dtype=str)

    ground_truth_left = ground_truth_content[1:16]
    ground_truth_right = ground_truth_content[16:-1]

    result_l, image_l = find_x(left_grayscale_image, cols_l, rows_l)
    #print(result_l)
    result_r, image_r = find_x(right_grayscale_image, cols_r, rows_r)
    #print(result_r)

    left_mis = 0;
    for r in range(15):
        tr_ans = ground_truth_left[r][1]
        if r + 1 not in result_l.keys():
            left_mis += 1
        else:
            if len(result_l[r+1]) > 1:
                left_mis += 1
            else:
                if result_l[r+1][0] != tr_ans:
                    left_mis += 1
                else:
                    number_ok += 1

    right_mis = 0;
    for r in range(15):
        tr_ans = ground_truth_right[r][1]
        if r + 1 not in result_r.keys():
            right_mis += 1
        else:
            if len(result_r[r+1]) > 1:
                right_mis += 1
            else:
                if result_r[r+1][0] != tr_ans:
                    right_mis += 1
                else:
                    number_ok += 1

    return (number_ok * 3) / 10 + 1

#6
acc = 0
for i in range(1, 151):
    image_name = 'perspective_' + str(i) + '.jpg'
   
    print(image_name)
    file = open(folder_in + 'image_' + str(i) +'.txt', 'r')
    first_line = file.readline()
    file.close() 
    first_line = first_line.replace('\n', '')
    info = first_line.split(' ')

    grade = predict_grade_scenario_2(folder_in + image_name, info[0], int(info[1]))

    if ground_truth_hw_dict[i] == str(grade):
        acc += 1
    else:
        print("GRESEALA")

print("Acc = " + str(acc * 100 / 150) + " %")

#7
acc = 0
for i in range(1, 151):
    image_name = 'rotation_' + str(i) + '.jpg'
   
    print(image_name)
    file = open(folder_in + 'image_' + str(i) +'.txt', 'r')
    first_line = file.readline()
    file.close() 
    first_line = first_line.replace('\n', '')
    info = first_line.split(' ')

    grade = predict_grade_scenario_2(folder_in + image_name, info[0], int(info[1]))

    if ground_truth_hw_dict[i] == str(grade):
        acc += 1
    else:
        print("GRESEALA")

print("Acc = " + str(acc * 100 / 150) + " %")

#8
acc = 0
for i in range(1, 151):
    image_name = 'image_' + str(i) + '.jpg'
   
    print(image_name)
    file = open(folder_in + 'image_' + str(i) +'.txt', 'r')
    first_line = file.readline()
    file.close() 
    first_line = first_line.replace('\n', '')
    info = first_line.split(' ')

    grade = predict_grade_scenario_2(folder_in + image_name, info[0], int(info[1]))

    if ground_truth_hw_dict[i] == str(grade):
        acc += 1
    else:
        print("GRESEALA")

print("Acc = " + str(acc * 100 / 150) + " %")

"""# **Scenario 3 (no annotations)**
You receive a test set containing 75 images (scanned, rotated or perspective view). There is no annotation available. For each image you have to output the corresponding grade
"""

#1
def detect_subject_choice(choice_patch_grayscale, rows, cols):

    # apply findCountour function in order to extract all the countour in the image
    # take only the black (ish) pixels
    mask = np.uint8(choice_patch_grayscale < 50)
    contours, hierarchy = cv.findContours(mask, cv.RETR_TREE, cv.CHAIN_APPROX_SIMPLE)
    
    areas = []
    coords = []
    if len(contours) != 0:
        for idx_det in range(len(contours)):
            contour = contours[idx_det] 
            contour = np.squeeze(contour)  
            
            if contour.ndim == 1:
                continue
                
            x_min = np.min(contour[:, 0])
            x_max = np.max(contour[:, 0])

            y_min = np.min(contour[:, 1])
            y_max = np.max(contour[:, 1])
            areas.append(((x_max - x_min) * (y_max - y_min)))
            coords.append([(x_min, y_min), (x_max, y_max)])
    if len(areas) < 2:
        return None
    
    areas = np.array(areas)
    indices = areas.argsort()
    bbox_1 = coords[indices[-1]]
    bbox_2 = coords[indices[-2]]

    choice_patch_color = np.dstack((choice_patch_grayscale, choice_patch_grayscale, choice_patch_grayscale))
    cv.rectangle(choice_patch_color, bbox_1[0], bbox_1[1], color=(255, 0, 255), thickness=5)
    cv.rectangle(choice_patch_color, bbox_2[0], bbox_2[1], color=(0, 255, 255), thickness=5)
    #cv2_imshow(choice_patch_color)

    return bbox_1, bbox_2

#2
def make_data_set():

    if not os.path.exists(folder_digits + '1'):
        os.makedirs(folder_digits + '1')
    if not os.path.exists(folder_digits + '2'):
        os.makedirs(folder_digits + '2')
    if not os.path.exists(folder_digits + '3'):
        os.makedirs(folder_digits + '3')
    if not os.path.exists(folder_digits + '4'):
        os.makedirs(folder_digits + '4')

    for f in glob.iglob(folder_digits + '1/*'):
        os.remove(f)
    for f in glob.iglob(folder_digits + '2/*'):
        os.remove(f)
    for f in glob.iglob(folder_digits + '3/*'):
        os.remove(f)
    for f in glob.iglob(folder_digits + '4/*'):
        os.remove(f)

    for i in range(1, 151): 
        file = open(folder_in + 'image_' + str(i) +'.txt', 'r')
        first_line = file.readline()
        file.close() 
        first_line = first_line.replace('\n', '')
        info = first_line.split(' ')

        image_name = 'image_' + str(i) + '.jpg'
    
        print(image_name)
        # load image
        image = cv.imread(folder_in + image_name)

        try:
            #aligned_image2 = transform_rotaded_and_perspective_in_scanned(folder_in + image_name)
            #print_img(aligned_image2)

            # take the lower part of the image
            #orig_h, orig_w, _ = aligned_image2.shape
            #image = aligned_image2[int(orig_h * 0):int(orig_h * 0.85)]

            orig_h, orig_w, _ = image.shape
            image = image[int(orig_h * 0.41):int(orig_h * 0.88)]

            img_blur = cv.GaussianBlur(image,(7,7),0)
            grayscale_image = cv.cvtColor(img_blur, cv.COLOR_BGR2GRAY)
            th = cv.adaptiveThreshold(grayscale_image,255,cv.ADAPTIVE_THRESH_GAUSSIAN_C, cv.THRESH_BINARY,11,2)

            left_grayscale_image = th[:, :orig_w//2]
            right_grayscale_image = th[:, orig_w//2:]

            _, bbox_l, cols_l, rows_l = find_table(left_grayscale_image)
            _, bbox_r, cols_r, rows_r = find_table(right_grayscale_image)

            x_min = cols_r[-2][0][0]
            x_max = cols_r[-1][0][0]
            y_min = 0 
            y_max = rows_r[0][1][1] - 2*(rows_r[1][1][1]-rows_r[0][1][1])

            patch = grayscale_image[:, orig_w//2:]
            patch = patch[y_min:y_max, x_min:x_max]

            bbox_1, bbox_2 = detect_subject_choice(patch, rows_r, cols_r)

            (x_min_1, y_min_1), (x_max_1, y_max_1) = bbox_1
            (x_min_2, y_min_2), (x_max_2, y_max_2) = bbox_2
            
            patch1 = patch[y_min_1 + 10:y_max_1 - 10, x_min_1 + 10:x_max_1 - 10]
            patch2 = patch[y_min_2 + 10:y_max_2 - 10, x_min_2 + 10:x_max_2 - 10]

            mean_patch_box_1 = np.round(patch1.mean())
            mean_patch_box_2 = np.round(patch2.mean())

            choice_patch_color = np.dstack((patch, patch, patch))
            if mean_patch_box_1 < mean_patch_box_2:
                cv.imwrite(folder_digits + info[1] + '/' + str(i) + '.jpg', patch1)
                cv.rectangle(choice_patch_color, (x_min_1 + 10, y_min_1 + 10), (x_max_1 - 10, y_max_1 - 10), color=(255, 0, 255), thickness=5)
            else:
                cv.imwrite(folder_digits + info[1] + '/' + str(i) + '.jpg', patch2)
                cv.rectangle(choice_patch_color, (x_min_2 + 10, y_min_2 + 10), (x_max_2 - 10, y_max_2 - 10), color=(0, 255, 255), thickness=5)
            #cv2_imshow(choice_patch_color)
            #print(info[0])
        except Exception as e:
            print(e)
            continue

#3
make_data_set()

#4
# Training settings    
kwargs={}
class Args():
  def __init__(self):
      self.batch_size = 64
      self.test_batch_size = 64
      self.epochs = 10
      self.lr = 0.01
      self.momentum = 0.9
      self.seed = 1
      self.log_interval = int(10000 / self.batch_size)
      self.cuda = False

args = Args()

use_cuda = torch.cuda.is_available()
torch.manual_seed(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")

kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}

#5
train_loader = torch.utils.data.DataLoader(
    datasets.MNIST('../data', train=True, download=True,
                   transform=transforms.Compose([
                       transforms.ToTensor(),
                       
                   ])),
    batch_size=args.batch_size, shuffle=True,drop_last=True, **kwargs)

test_loader = torch.utils.data.DataLoader(
    datasets.MNIST('../data', train=False, transform=transforms.Compose([
                       transforms.ToTensor(),
                   ])),
    batch_size=args.test_batch_size, shuffle=True,drop_last=True, **kwargs)

#6
no_filters1 = 20
no_filter2 = 50
no_neurons1 = 500
class CNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, no_filters1, 5, 1)
        self.conv2 = nn.Conv2d(no_filters1, no_filter2, 5, 1)
        self.fc1 = nn.Linear(4*4*no_filter2, no_neurons1)
        self.fc2 = nn.Linear(no_neurons1, 10)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.max_pool2d(x, 2, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2, 2)
        x = x.view(-1, 4*4*no_filter2)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)

#7
def train(args, model, device, train_loader, optimizer, epoch):
    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()
        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.item()))

def test(args, model, device, test_loader):
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss
            pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)

    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))

#8
model = CNN().to(device)

optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)

for epoch in range(1, args.epochs + 1):
    train(args, model, device, train_loader, optimizer, epoch)
    test(args, model, device, test_loader)

torch.save(model.state_dict(), folder_models + "mnist_cnn.pt")

#9
model = CNN().to(device)
model.load_state_dict(torch.load(folder_models + "mnist_cnn.pt", map_location=torch.device(device)))

#10
total = 0
acc = 0
with torch.no_grad():
    for img_name in glob.iglob(folder_digits + '1/*'):
        total += 1
        #print(img_name)

        img = cv.imread(img_name, cv.IMREAD_GRAYSCALE)
        #img = img[5:-5, 5:-5]
        img = cv.GaussianBlur(img, (7, 7), 0)
        th = cv.adaptiveThreshold(img,255,cv.ADAPTIVE_THRESH_MEAN_C, cv.THRESH_BINARY,7,2)
        th = cv.bitwise_not(th)
        th = th [5:-5, 5:-5]
        th = cv.copyMakeBorder(th, 10, 10, 10, 10, cv.BORDER_CONSTANT, None, 0)
        #cv2_imshow(th)
        resized = cv.resize(th, (28, 28), interpolation = cv.INTER_AREA)
        resized = np.array(resized, dtype = float)
        data = torch.from_numpy(np.expand_dims(np.expand_dims(resized, 0), 0))
        data = data.type(torch.FloatTensor)

        data = data.to(device)
        output = model(data)
        output = output[0].cpu().numpy()
        output = output[1:5]
        #print(output)

        pred = output.argmax() + 1 # get the index of the max log-probability
        #print(pred)
        if pred == 1:
            acc += 1
        elif pred == 4 and output[3] - output[0] < 2000:
            acc += 1
        else:
            cv2_imshow(th)
            print(pred)
            print(output)

print(acc * 100 / total)

#11
total = 0
acc = 0
diffs = []
with torch.no_grad():
    for img_name in glob.iglob(folder_digits + '2/*'):
        total += 1
        #print(img_name)

        img = cv.imread(img_name, cv.IMREAD_GRAYSCALE)
        #img = img[5:-5, 5:-5]
        img = cv.GaussianBlur(img, (7, 7), 0)
        th = cv.adaptiveThreshold(img,255,cv.ADAPTIVE_THRESH_MEAN_C, cv.THRESH_BINARY,7,2)
        th = cv.bitwise_not(th)
        th = th [5:-5, 5:-5]
        th = cv.copyMakeBorder(th, 10, 10, 10, 10, cv.BORDER_CONSTANT, None, 0)
        #cv2_imshow(th)
        resized = cv.resize(th, (28, 28), interpolation = cv.INTER_AREA)
        resized = np.array(resized, dtype = float)
        data = torch.from_numpy(np.expand_dims(np.expand_dims(resized, 0), 0))
        data = data.type(torch.FloatTensor)

        data = data.to(device)
        output = model(data)
        output = output[0].cpu().numpy()
        output = output[1:5]
        #print(output)

        pred = output.argmax() + 1 # get the index of the max log-probability
        #print(pred)
        if pred == 2:
            acc += 1
        else:
            cv2_imshow(th)
            print(pred)
            print(output)

print(acc * 100 / total)

#12
total = 0
acc = 0
with torch.no_grad():
    for img_name in glob.iglob(folder_digits + '3/*'):
        total += 1
        #print(img_name)

        img = cv.imread(img_name, cv.IMREAD_GRAYSCALE)
        #img = img[5:-5, 5:-5]
        img = cv.GaussianBlur(img, (7, 7), 0)
        th = cv.adaptiveThreshold(img,255,cv.ADAPTIVE_THRESH_MEAN_C, cv.THRESH_BINARY,7,2)
        th = cv.bitwise_not(th)
        th = th [5:-5, 5:-5]
        th = cv.copyMakeBorder(th, 10, 10, 10, 10, cv.BORDER_CONSTANT, None, 0)
        #cv2_imshow(th)
        resized = cv.resize(th, (28, 28), interpolation = cv.INTER_AREA)
        resized = np.array(resized, dtype = float)
        data = torch.from_numpy(np.expand_dims(np.expand_dims(resized, 0), 0))
        data = data.type(torch.FloatTensor)

        data = data.to(device)
        output = model(data)
        output = output[0].cpu().numpy()
        output = output[1:5]
        #print(output)

        pred = output.argmax() + 1 # get the index of the max log-probability
        #print(pred)
        if pred == 3:
            acc += 1
        else:
            cv2_imshow(th)
            print(pred)
            print(output)

print(acc * 100 / total)

#13
total = 0
acc = 0
with torch.no_grad():
    for img_name in glob.iglob(folder_digits + '4/*'):
        total += 1
        #print(img_name)

        img = cv.imread(img_name, cv.IMREAD_GRAYSCALE)
        #img = img[5:-5, 5:-5]
        img = cv.GaussianBlur(img, (7, 7), 0)
        th = cv.adaptiveThreshold(img,255,cv.ADAPTIVE_THRESH_MEAN_C, cv.THRESH_BINARY,7,2)
        th = cv.bitwise_not(th)
        th = th [5:-5, 5:-5]
        th = cv.copyMakeBorder(th, 10, 10, 10, 10, cv.BORDER_CONSTANT, None, 0)
        #cv2_imshow(th)
        resized = cv.resize(th, (28, 28), interpolation = cv.INTER_AREA)
        resized = np.array(resized, dtype = float)
        data = torch.from_numpy(np.expand_dims(np.expand_dims(resized, 0), 0))
        data = data.type(torch.FloatTensor)

        data = data.to(device)
        output = model(data)
        output = output[0].cpu().numpy()
        output = output[1:5]
        #print(output)

        pred = output.argmax() + 1 # get the index of the max log-probability
        #print(pred)
        if pred == 4 and output[3] - output[0] > 2000:
            acc += 1
        else:
            cv2_imshow(th)
            print(pred)
            print(output)

print(acc * 100 / total)

#14
def eval_patch(patch):
    with torch.no_grad():

        (h, w) = patch.shape[:2]
        if h == 0 or w == 0:
            return None
        width = 90
        r = width / float(w)
        dim = (width, int(h * r))
        patch = cv.resize(patch, dim, interpolation = cv.INTER_AREA)

        #img = img[5:-5, 5:-5]
        img = cv.GaussianBlur(patch, (7, 7), 0)
        th = cv.adaptiveThreshold(img,255,cv.ADAPTIVE_THRESH_MEAN_C, cv.THRESH_BINARY,7,2)
        th = cv.bitwise_not(th)
        th = th [5:-5, 5:-5]
        th = cv.copyMakeBorder(th, 10, 10, 10, 10, cv.BORDER_CONSTANT, None, 0)
        #cv2_imshow(th)
        resized = cv.resize(th, (28, 28), interpolation = cv.INTER_AREA)
        resized = np.array(resized, dtype = float)
        data = torch.from_numpy(np.expand_dims(np.expand_dims(resized, 0), 0))
        data = data.type(torch.FloatTensor)

        data = data.to(device)
        output = model(data)
        output = output[0].cpu().numpy()
        output = output[1:5]
        #print(output)

        pred = output.argmax() + 1 # get the index of the max log-probability
        #print(pred)
        if pred == 4 and output[3] - output[0] < 2000:
            pred = 1
        return pred

#15
def set_global_cols_and_rows_for_scenario_3():
    img_template = cv.imread(data_dir_drive + '/data/template.jpg')
    (h, w) = img_template.shape[:2]
    width = 3500
    r = width / float(w)
    dim = (width, int(h * r))
    img_template = cv.resize(img_template, dim, interpolation = cv.INTER_AREA)
    orig_h, orig_w, _ = img_template.shape
    img_template = img_template[int(orig_h * 0):int(orig_h * 0.83)]
    img_template = img_template[:, int(orig_w * 0.05):int(orig_w * 0.95)]

    #print_img(img_template)
    img_template = cv.GaussianBlur(img_template,(7,7),0)
    img_template = cv.cvtColor(img_template, cv.COLOR_BGR2GRAY)

    left_img_template = img_template[:, :orig_w//2]
    right_img_template = img_template[:, orig_w//2:]

    _, _, cols_l, rows_l = find_table(left_img_template)
    _, _, cols_r, rows_r = find_table(right_img_template)

    return cols_l, rows_l, cols_r, rows_r

cols_l, rows_l, cols_r, rows_r = set_global_cols_and_rows_for_scenario_3()

#16
def predict_grade_scenario_3(path):
    cols_l, rows_l, cols_r, rows_r = set_global_cols_and_rows_for_scenario_3()

    aligned_image2 = transform_rotaded_and_perspective_in_scanned(path)
    #print_img(aligned_image2)

    # take the lower part of the image
    orig_h, orig_w, _ = aligned_image2.shape
    aligned_image2 = aligned_image2[int(orig_h * 0):int(orig_h * 0.83)]
    image = aligned_image2[:, int(orig_w * 0.05):int(orig_w * 0.95)]

    #print_img(image)
    img_blur = cv.GaussianBlur(image,(7,7),0)
    grayscale_image = cv.cvtColor(img_blur, cv.COLOR_BGR2GRAY)
    th = cv.adaptiveThreshold(grayscale_image,255,cv.ADAPTIVE_THRESH_GAUSSIAN_C, cv.THRESH_BINARY,11,2)

    left_grayscale_image = th[:, :orig_w//2]
    right_grayscale_image = th[:, orig_w//2:]

    result_l, image_l = find_x(left_grayscale_image, cols_l, rows_l)
    #print(result_l)
    result_r, image_r = find_x(right_grayscale_image, cols_r, rows_r)
    #print(result_r)

    x_min = cols_r[-2][0][0]
    x_max = cols_r[-1][0][0]
    y_min = 0 
    y_max = rows_r[0][1][1] - 2*(rows_r[1][1][1]-rows_r[0][1][1]) # 2 randuri in sus

    patch = grayscale_image[:, orig_w//2:]
    patch = patch[y_min:y_max, x_min:x_max]
    #print_img(patch)

    bbox_1, bbox_2 = detect_subject_choice(patch, rows_r, cols_r)

    (x_min_1, y_min_1), (x_max_1, y_max_1) = bbox_1
    (x_min_2, y_min_2), (x_max_2, y_max_2) = bbox_2
            
    patch1 = patch[y_min_1 + 10:y_max_1 - 10, x_min_1 + 10:x_max_1 - 10]
    patch2 = patch[y_min_2 + 10:y_max_2 - 10, x_min_2 + 10:x_max_2 - 10]

    if y_min_1 > y_min_2:
        patch1, patch2 = patch2, patch1

    mean_patch_box_1 = np.round(patch1.mean())
    mean_patch_box_2 = np.round(patch2.mean())

    subject, number = 0, 0
    choice_patch_color = np.dstack((patch, patch, patch))
    if mean_patch_box_1 < mean_patch_box_2: #Info
        subject = 'I'
        number = eval_patch(patch1)
        cv.rectangle(choice_patch_color, (x_min_1 + 10, y_min_1 + 10), (x_max_1 - 10, y_max_1 - 10), color=(255, 0, 255), thickness=5)
    else: #Fizica
        subject = 'F'
        number = eval_patch(patch2)
        cv.rectangle(choice_patch_color, (x_min_2 + 10, y_min_2 + 10), (x_max_2 - 10, y_max_2 - 10), color=(0, 255, 255), thickness=5)

    #print(subject)
    #print(number)

    number_ok = 0
    # load ground truth correct answers / barem
    if subject == 'F':
        if number == 1:
            ground_truth_content = np.loadtxt(folder_barem + 'Fizica_varianta1.txt', dtype=str)
        elif number == 2:
            ground_truth_content = np.loadtxt(folder_barem + 'Fizica_varianta2.txt', dtype=str)
        elif number == 3:
            ground_truth_content = np.loadtxt(folder_barem + 'Fizica_varianta3.txt', dtype=str)
        else:
            ground_truth_content = np.loadtxt(folder_barem + 'Fizica_varianta4.txt', dtype=str)
    else:
        if number == 1:
            ground_truth_content = np.loadtxt(folder_barem + 'Informatica_varianta1.txt', dtype=str)
        elif number == 2:
            ground_truth_content = np.loadtxt(folder_barem + 'Informatica_varianta2.txt', dtype=str)
        elif number == 3:
            ground_truth_content = np.loadtxt(folder_barem + 'Informatica_varianta3.txt', dtype=str)
        else:
            ground_truth_content = np.loadtxt(folder_barem + 'Informatica_varianta4.txt', dtype=str)

    ground_truth_left = ground_truth_content[1:16]
    ground_truth_right = ground_truth_content[16:-1]

    left_mis = 0;
    for r in range(15):
        tr_ans = ground_truth_left[r][1]
        if r + 1 not in result_l.keys():
            left_mis += 1
        else:
            if len(result_l[r+1]) > 1:
                left_mis += 1
            else:
                if result_l[r+1][0] != tr_ans:
                    left_mis += 1
                else:
                    number_ok += 1

    right_mis = 0;
    for r in range(15):
        tr_ans = ground_truth_right[r][1]
        if r + 1 not in result_r.keys():
            right_mis += 1
        else:
            if len(result_r[r+1]) > 1:
                right_mis += 1
            else:
                if result_r[r+1][0] != tr_ans:
                    right_mis += 1
                else:
                    number_ok += 1

    return (number_ok * 3) / 10 + 1, subject, number, th

#17
acc = 0
for i in range(1, 151):
    image_name = 'image_' + str(i) + '.jpg'
   
    print(image_name)
    file = open(folder_in + 'image_' + str(i) +'.txt', 'r')
    first_line = file.readline()
    file.close() 
    first_line = first_line.replace('\n', '')
    info = first_line.split(' ')

    try:
        grade, subject, number, th = predict_grade_scenario_3(folder_in + image_name)

        if ground_truth_hw_dict[i] == str(grade) and subject == info[0] and number == int(info[1]):
            acc += 1
        else:
            print("GRESEALA")
            print( grade )
            print( subject )
            print( number )
            print_img( th )

    except Exception as e:
        print(e)
        continue

print("Acc = " + str(acc * 100 / 150) + " %")

#18
acc = 0
for i in range(1, 151):
    image_name = 'perspective_' + str(i) + '.jpg'
   
    print(image_name)
    file = open(folder_in + 'image_' + str(i) +'.txt', 'r')
    first_line = file.readline()
    file.close() 
    first_line = first_line.replace('\n', '')
    info = first_line.split(' ')

    try:
        grade, subject, number, th = predict_grade_scenario_3(folder_in + image_name)

        if ground_truth_hw_dict[i] == str(grade) and subject == info[0] and number == int(info[1]):
            acc += 1
        else:
            print("GRESEALA")
            print( grade )
            print( subject )
            print( number )
            print_img( th )

    except Exception as e:
        print(e)
        continue

print("Acc = " + str(acc * 100 / 150) + " %")

#19
acc = 0
for i in range(1, 151):
    image_name = 'rotation_' + str(i) + '.jpg'
   
    print(image_name)
    file = open(folder_in + 'image_' + str(i) +'.txt', 'r')
    first_line = file.readline()
    file.close() 
    first_line = first_line.replace('\n', '')
    info = first_line.split(' ')

    grade, subject, number, th = predict_grade_scenario_3(folder_in + image_name)

    try:
        grade, subject, number, th = predict_grade_scenario_3(folder_in + image_name)

        if ground_truth_hw_dict[i] == str(grade) and subject == info[0] and number == int(info[1]):
            acc += 1
        else:
            print("GRESEALA")
            print( grade )
            print( subject )
            print( number )
            print_img( th )

    except Exception as e:
        print(e)
        continue

print("Acc = " + str(acc * 100 / 150) + " %")

"""# **Scenario 4 (handwritten recognition)**
Your receive a test set containing 25 scanned images. We will make sure to modify some content such that the grade written in red to be different that the grade obtained by a perfect grading system. For each image you have to output the corresponding handwritten grade.
"""

#1
hack = {1:[0,3,6,9],
        2:[2,5,8],
        3:[1,4,7],
        4:[0,3,6,9],
        5:[2,5,8],
        6:[1,4,7],
        7:[0,3,6,9],
        8:[2,5,8],
        9:[1,4,7]}

#2
def cut_background(image, proj):
    w, h = image.shape[::-1]
    left = 0
    ref_sum_img = np.sum(image, axis = 0)
    ref_sum_img = ref_sum_img //255
    in_0_region = False
    count_in_0_region = 0
    line = 0
    
    for line in range(int(len(ref_sum_img) * proj)):
        if ref_sum_img[line] < 5:
            if not in_0_region:
                count_in_0_region = 0
                in_0_region = True
            else:
                count_in_0_region += 1
        else:
            in_0_region = False
            if count_in_0_region >= 10:
                left = line - 1
                break

    if count_in_0_region >= 10:
        left = line - 1
    return image[:, left:]

#3
def predict_first_digit(image):
    w, h = image.shape[::-1]
    img_model = image[:, :int(0.4 * w)]
    img_model = cv.copyMakeBorder(img_model, 1, 1, 1, 1, cv.BORDER_CONSTANT, None, 0)

    w, h = img_model.shape[::-1]

    if w > h:
        diff = w - h
        img_model = cv.copyMakeBorder(img_model, diff // 2, diff - diff//2, 0, 0, cv.BORDER_CONSTANT, None, 0)

    if w < h:
        diff = h - w
        img_model = cv.copyMakeBorder(img_model, 0, 0, diff // 2, diff - diff//2, cv.BORDER_CONSTANT, None, 0)

    cv2_imshow(img_model)
    resized = cv.resize(img_model, (28, 28), interpolation = cv.INTER_AREA)
    resized = np.array(resized, dtype = float)
    data = torch.from_numpy(np.expand_dims(np.expand_dims(resized, 0), 0))
    data = data.type(torch.FloatTensor)

    data = data.to(device)
    output = model(data)
    output = output[0].detach().cpu().numpy()
    #print(output)
    output = output[3:]
    pred = output.argmax() + 3 # get the index of the max log-probability
    return pred

#4
def predict_second_digit(image, pred1):
    w, h = image.shape[::-1]
    img_model = image[:, int(0.5 * w):int(0.85 * w)]
    img_model = cv.copyMakeBorder(img_model, 1, 1, 1, 1, cv.BORDER_CONSTANT, None, 0)

    w, h = img_model.shape[::-1]

    if w > h:
        diff = w - h
        img_model = cv.copyMakeBorder(img_model, diff // 2, diff - diff//2, 0, 0, cv.BORDER_CONSTANT, None, 0)

    if w < h:
        diff = h - w
        img_model = cv.copyMakeBorder(img_model, 0, 0, diff // 2, diff - diff//2, cv.BORDER_CONSTANT, None, 0)

    cv2_imshow(img_model)
    resized = cv.resize(img_model, (28, 28), interpolation = cv.INTER_AREA)
    resized = np.array(resized, dtype = float)
    data = torch.from_numpy(np.expand_dims(np.expand_dims(resized, 0), 0))
    data = data.type(torch.FloatTensor)

    data = data.to(device)
    output = model(data)
    output = output[0].detach().cpu().numpy()

    #print(output)
    #print(hack[pred1])
    output2 = output[hack[pred1]]
    #print(output2)
    pred = 3 * np.argmax(output2)

    if pred1 == 3 or pred1 == 6 or pred1 == 9:
        pred += 1
    if pred1 == 2 or pred1 == 5 or pred1 == 8:
        pred += 2

    return pred

#5
def make_data_set_handwritten():

    if not os.path.exists(folder_digits + 'handwritten'):
        os.makedirs(folder_digits + 'handwritten')

    for f in glob.iglob(folder_digits + 'handwritten/*'):
        os.remove(f)

    perm = np.random.permutation(range(1, 151))
    acc = 0
    for i in range(1, 151): 
        file = open(folder_in + 'image_' + str(i) +'.txt', 'r')
        first_line = file.readline()
        file.close() 
        first_line = first_line.replace('\n', '')
        info = first_line.split(' ')

        image_name = 'image_' + str(i) + '.jpg'
    
        print(image_name)
        # load image
        image = cv.imread(folder_in + image_name)

        try:
            orig_h, orig_w, _ = image.shape
            image = image[int(orig_h * 0.29):int(orig_h * 0.33)]
            image = image[:, int(orig_w * 0.08):int(orig_w * 0.29)]
            orig_h, orig_w, _ = image.shape

            img = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
            img = cv.GaussianBlur(img, (11, 11), 0)
            th = cv.adaptiveThreshold(img,255,cv.ADAPTIVE_THRESH_GAUSSIAN_C, cv.THRESH_BINARY,7,2)
            th = cv.bitwise_not(th)
            #cv2_imshow(th)

            template = cv.imread(data_dir_drive + '/template_matching.JPG',0)
            w, h = template.shape[::-1]
            img = th.copy()
            left_cut_size = int(orig_w * 0.25)
            right_cut_size = int(orig_w * 0.55)
            img = img[:, left_cut_size:right_cut_size]
            method = eval('cv.TM_CCOEFF_NORMED')
            # Apply template Matching
            res = cv.matchTemplate(img,template, cv.TM_CCOEFF_NORMED)
            min_val, max_val, min_loc, max_loc = cv.minMaxLoc(res)
            top_left = (max_loc[0] + left_cut_size, max_loc[1])
            bottom_right = (top_left[0] + w, top_left[1] + h)
            #cv.rectangle(th,top_left, bottom_right, 255, 2)
            #cv2_imshow(th)
            new_th = th[:, :max_loc[0] + left_cut_size]
            #cv2_imshow(new_th)
        
            template = cv.imread(data_dir_drive + '/template_matching2.JPG',0)
            w, h = template.shape[::-1]
            img = new_th.copy()
            method = eval('cv.TM_CCOEFF_NORMED')
            # Apply template Matching
            res = cv.matchTemplate(img,template, cv.TM_CCOEFF_NORMED)
            min_val, max_val, min_loc, max_loc = cv.minMaxLoc(res)
            top_left = max_loc
            bottom_right = (top_left[0] + w, top_left[1] + h)
            #cv.rectangle(new_th,top_left, bottom_right, 255, 2)
            #cv2_imshow(new_th)
            new_th = new_th[:top_left[1], :]
            #cv2_imshow(new_th)

            new_th = cut_background(new_th, 0.5)
            new_th = np.transpose(cut_background(np.transpose(new_th), 0.5))
            new_th = np.fliplr(cut_background(np.fliplr(new_th), 0.1))
            new_th = np.flipud(cut_background(np.flipud(new_th), 0.1))
            #cv2_imshow(new_th)

            pred1 = predict_first_digit(new_th)
            #print(pred1)
            pred2 = predict_second_digit(new_th, pred1)
            #print(pred2)

            print("RASPUNS = " + str((10 * pred1 + pred2) / 10))
            print("TRUE = " + ground_truth_hw_dict[i])
            if ground_truth_hw_dict[i] == str((10 * pred1 + pred2) / 10):
                acc += 1

        except Exception as e:
            print(e)
            
    print("ACC = " + str(acc / 150))

make_data_set_handwritten()

#6
def predict_grade_scenario_4(path):
    try:
        image = cv.imread(path)
        orig_h, orig_w, _ = image.shape
        image = image[int(orig_h * 0.29):int(orig_h * 0.33)]
        image = image[:, int(orig_w * 0.08):int(orig_w * 0.29)]
        orig_h, orig_w, _ = image.shape

        img = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
        img = cv.GaussianBlur(img, (11, 11), 0)
        th = cv.adaptiveThreshold(img,255,cv.ADAPTIVE_THRESH_GAUSSIAN_C, cv.THRESH_BINARY,7,2)
        th = cv.bitwise_not(th)
        #cv2_imshow(th)

        template = cv.imread(data_dir_drive + '/template_matching.JPG',0)
        w, h = template.shape[::-1]
        img = th.copy()
        left_cut_size = int(orig_w * 0.25)
        right_cut_size = int(orig_w * 0.55)
        img = img[:, left_cut_size:right_cut_size]
        method = eval('cv.TM_CCOEFF_NORMED')
        # Apply template Matching
        res = cv.matchTemplate(img,template, cv.TM_CCOEFF_NORMED)
        min_val, max_val, min_loc, max_loc = cv.minMaxLoc(res)
        top_left = (max_loc[0] + left_cut_size, max_loc[1])
        bottom_right = (top_left[0] + w, top_left[1] + h)
        #cv.rectangle(th,top_left, bottom_right, 255, 2)
        #cv2_imshow(th)
        new_th = th[:, :max_loc[0] + left_cut_size]
        #cv2_imshow(new_th)
        
        template = cv.imread(data_dir_drive + '/template_matching2.JPG',0)
        w, h = template.shape[::-1]
        img = new_th.copy()
        method = eval('cv.TM_CCOEFF_NORMED')
        # Apply template Matching
        res = cv.matchTemplate(img,template, cv.TM_CCOEFF_NORMED)
        min_val, max_val, min_loc, max_loc = cv.minMaxLoc(res)
        top_left = max_loc
        bottom_right = (top_left[0] + w, top_left[1] + h)
        #cv.rectangle(new_th,top_left, bottom_right, 255, 2)
        #cv2_imshow(new_th)
        new_th = new_th[:top_left[1], :]
        #cv2_imshow(new_th)

        new_th = cut_background(new_th, 0.5)
        new_th = np.transpose(cut_background(np.transpose(new_th), 0.5))
        new_th = np.fliplr(cut_background(np.fliplr(new_th), 0.1))
        new_th = np.flipud(cut_background(np.flipud(new_th), 0.1))
        #cv2_imshow(new_th)

        pred1 = predict_first_digit(new_th)
        #print(pred1)
        pred2 = predict_second_digit(new_th, pred1)
        #print(pred2)

        return (10 * pred1 + pred2) / 10

    except Exception as e:
        print(e)

#7
acc = 0
for i in range(1, 151): 
    image_name = 'image_' + str(i) + '.jpg'
    print(image_name)
    # load image
    grade = predict_grade_scenario_4(folder_in + image_name)

    print("RASPUNS = " + str(grade))
    print("TRUE = " + ground_truth_hw_dict[i])
    if ground_truth_hw_dict[i] == str(grade):
        acc += 1

print("ACC = " + str(acc / 150))

"""# **Running the entire code**"""

#1
def run_scenario_1():
    ans = {}
    for f in glob.iglob(input_folder_scanned + '*.jpg'):
        aux = f.replace('.jpg', '')[-2:]
        subject, number = aux[0], int(aux[1])
        grade = predict_grade_scenario_1(f, subject, number)
        ans[f.replace(input_folder_scanned, '')] = grade
    
    f = open(output_folder + 'lupascu_marian_407_task1.txt', 'w')
    for k, v in ans.items():
        f.write("%s\t%2.2f\n" % (k, v))
    f.close()

#2
def run_scenario_2():
    ans = {}
    for f in glob.iglob(input_folder_rotated_and_perspective + '*.jpg'):
        aux = f.replace('.jpg', '')[-2:]
        subject, number = aux[0], int(aux[1])
        grade = predict_grade_scenario_2(f, subject, number)
        ans[f.replace(input_folder_rotated_and_perspective, '')] = grade
    
    f = open(output_folder + 'lupascu_marian_407_task2.txt', 'w')
    for k, v in ans.items():
        f.write("%s\t%2.2f\n" % (k, v))
    f.close()

#3
def run_scenario_3():
    ans = {}
    for f in glob.iglob(input_folder_no_annotation + '*.jpg'):
        grade, _, _, _ = predict_grade_scenario_3(f)
        ans[f.replace(input_folder_no_annotation, '')] = grade
    
    f = open(output_folder + 'lupascu_marian_407_task3.txt', 'w')
    for k, v in ans.items():
        f.write("%s\t%2.2f\n" % (k, v))
    f.close()

#4
def run_scenario_4():
    ans = {}
    for f in glob.iglob(input_folder_handwritten + '*.jpg'):
        grade = predict_grade_scenario_4(f)
        ans[f.replace(input_folder_handwritten, '')] = grade
    
    f = open(output_folder + 'lupascu_marian_407_task4.txt', 'w')
    for k, v in ans.items():
        f.write("%s\t%2.2f\n" % (k, v))
    f.close()

#5
def run():
    for f in glob.iglob(output_folder + '*'):
        os.remove(f)

    run_scenario_1()
    run_scenario_2()
    run_scenario_3()
    run_scenario_4()

#6
run()

#7
run_scenario_1()

#8
run_scenario_2()

#9
run_scenario_3()

#10
run_scenario_4()