preprocess.py

from __future__ import print_function
import os
import argparse
import torch
import torch.backends.cudnn as cudnn
import numpy as np
from align.data import cfg_mnet, cfg_re50
from align.layers.functions.prior_box import PriorBox
from align.utils.nms.py_cpu_nms import py_cpu_nms
import cv2
from align.models.retinaface import RetinaFace
from align.utils.box_utils import decode, decode_landm
import time
import skimage
from skimage import transform
from tqdm import tqdm
import json
from pathlib import Path

parser = argparse.ArgumentParser(description='Retinaface')

parser.add_argument('-m', '--trained_model', default='./PRETRAIN/ALIGN/Resnet50_Final.pth',
                    type=str, help='Trained state_dict file path to open')
parser.add_argument('--network', default='resnet50', help='Backbone network mobile0.25 or resnet50')
parser.add_argument('--cpu', action="store_true", default=False, help='Use cpu inference')
parser.add_argument('--confidence_threshold', default=0.02, type=float, help='confidence_threshold')
parser.add_argument('--top_k', default=5000, type=int, help='top_k')
parser.add_argument('--nms_threshold', default=0.4, type=float, help='nms_threshold')
parser.add_argument('--keep_top_k', default=750, type=int, help='keep_top_k')
parser.add_argument('-s', '--save_image', action="store_true", default=True, help='show detection results')
parser.add_argument('--vis_thres', default=0.6, type=float, help='visualization_threshold')
args = parser.parse_args()


def check_keys(model, pretrained_state_dict):
    ckpt_keys = set(pretrained_state_dict.keys())
    model_keys = set(model.state_dict().keys())
    used_pretrained_keys = model_keys & ckpt_keys
    unused_pretrained_keys = ckpt_keys - model_keys
    missing_keys = model_keys - ckpt_keys
    print('Missing keys:{}'.format(len(missing_keys)))
    print('Unused checkpoint keys:{}'.format(len(unused_pretrained_keys)))
    print('Used keys:{}'.format(len(used_pretrained_keys)))
    assert len(used_pretrained_keys) > 0, 'load NONE from pretrained checkpoint'
    return True


def remove_prefix(state_dict, prefix):
    ''' Old style model is stored with all names of parameters sharing common prefix 'module.' '''
    print('remove prefix \'{}\''.format(prefix))
    f = lambda x: x.split(prefix, 1)[-1] if x.startswith(prefix) else x
    return {f(key): value for key, value in state_dict.items()}


def load_model(model, pretrained_path, load_to_cpu):
    print('Loading pretrained model from {}'.format(pretrained_path))
    if load_to_cpu:
        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage)
    else:
        device = torch.cuda.current_device()
        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage.cuda(device))
    if "state_dict" in pretrained_dict.keys():
        pretrained_dict = remove_prefix(pretrained_dict['state_dict'], 'module.')
    else:
        pretrained_dict = remove_prefix(pretrained_dict, 'module.')
    check_keys(model, pretrained_dict)
    model.load_state_dict(pretrained_dict, strict=False)
    return model

IMSIZE = 317
def get_face(net, ori_img, save_name):
    #image_path = "./data/5.png"
    #ori_img = cv2.imread(image_path, cv2.IMREAD_COLOR)
    height, width, _ = ori_img.shape
    
    if height > IMSIZE*4 or width > IMSIZE*4:
        max_edge = max(height, width)
        height = height / (max_edge / (IMSIZE*4))
        width = width / (max_edge / (IMSIZE*4))
        height = int(height)
        width = int(width)
    if height < IMSIZE and width < IMSIZE :
        height = height * 2
        width = width * 2
    #img = img.resize((height, width),Image.BILINEAR)
    ori_img = cv2.resize(ori_img, (width,height), cv2.INTER_LINEAR)
    img_raw = ori_img.copy()

    std_points = np.array([[85.82991, 115.7792],
                            [169.0532, 114.3381],
                            [127.574, 167.0006],
                            [90.6964, 204.7014],
                            [167.3069, 203.3733]])
    padding = 30
    std_points = std_points + padding
    std_points = (std_points / 317) * IMSIZE

    img = np.float32(img_raw)

    im_height, im_width, _ = img.shape
    #img = np.flip(img,2).copy()
    scale = torch.Tensor([img.shape[1], img.shape[0], img.shape[1], img.shape[0]])
    img -= (104, 117, 123)
    img = img.transpose(2, 0, 1)
    img = torch.from_numpy(img).unsqueeze(0)
    img = img.to(device)
    scale = scale.to(device)

    tic = time.time()
    loc, conf, landms = net(img)  # forward pass
    #print('net forward time: {:.4f}'.format(time.time() - tic))

    priorbox = PriorBox(cfg, image_size=(im_height, im_width))
    priors = priorbox.forward()
    priors = priors.to(device)
    prior_data = priors.data
    boxes = decode(loc.data.squeeze(0), prior_data, cfg['variance'])
    boxes = boxes * scale / resize
    boxes = boxes.cpu().numpy()
    scores = conf.squeeze(0).data.cpu().numpy()[:, 1]
    landms = decode_landm(landms.data.squeeze(0), prior_data, cfg['variance'])
    scale1 = torch.Tensor([img.shape[3], img.shape[2], img.shape[3], img.shape[2],
                           img.shape[3], img.shape[2], img.shape[3], img.shape[2],
                           img.shape[3], img.shape[2]])
    scale1 = scale1.to(device)
    landms = landms * scale1 / resize
    landms = landms.cpu().numpy()

    # ignore low scores
    inds = np.where(scores > args.confidence_threshold)[0]
    boxes = boxes[inds]
    landms = landms[inds]
    scores = scores[inds]

    # keep top-K before NMS
    order = scores.argsort()[::-1][:args.top_k]
    boxes = boxes[order]
    landms = landms[order]
    scores = scores[order]

    # do NMS
    dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False)
    keep = py_cpu_nms(dets, args.nms_threshold)
    # keep = nms(dets, args.nms_threshold,force_cpu=args.cpu)
    dets = dets[keep, :]
    landms = landms[keep]

    # keep top-K faster NMS
    dets = dets[:args.keep_top_k, :]
    landms = landms[:args.keep_top_k, :]
    if len(landms) > 0:
        fd = landms[0].reshape(5,2)
        sim_trans = skimage.transform.estimate_transform('similarity', dst=fd, src=std_points)
        face = skimage.transform.warp(ori_img, sim_trans, output_shape=(IMSIZE, IMSIZE, 3))
        face = (face*255).astype(np.uint8)
        cv2.imwrite(save_name, face)
        return True
    else:
        return False

if __name__ == '__main__':
    torch.set_grad_enabled(False)
    cfg = None
    if args.network == "mobile0.25":
        cfg = cfg_mnet
    elif args.network == "resnet50":
        cfg = cfg_re50
    cfg['pretrain'] = False
    net = RetinaFace(cfg=cfg, phase = 'test')
    net = load_model(net, args.trained_model, args.cpu)
    net.eval()
    print('Finished loading model!')
    cudnn.benchmark = True
    device = torch.device("cpu" if args.cpu else "cuda")
    net = net.to(device)

    resize = 1
    save_idx = 0
    input_path = './DATASET/input'
    output_path = './DATASET/align_output'
    json_path = './DATASET/paths'
    video_list = os.listdir(input_path)
    paths = []
    for video in tqdm(video_list):
        video_capture = cv2.VideoCapture(os.path.join(input_path, video))
        frame_count = 0
        while True:
            ret, frame = video_capture.read()
            if not ret:
                break
            frame_count += 1
            if frame_count % 15 != 0:
                continue
            img_path = os.path.join(tar_path, str(save_idx)+'.png')
            find_face = get_face(net, frame, img_path)
            if find_face:
                save_idx += 1
                paths.append(img_path)
    json.dump(paths, open('./DATASET/paths/out.json', 'w'))