Skip to content

Commit

Permalink
- Bugfix: wrong access options when creating folders
Browse files Browse the repository at this point in the history 
- New auto rotation of horizontal faces, fixing bad landmark positions (expanded on ![PR 364](#364))
- Simple VR Option for stereo Images/Movies, best used in selected face mode
- Added RestoreFormer Enhancer - https://github.com/wzhouxiff/RestoreFormer
- Bumped up package versions for onnx/Torch etc.
  • Loading branch information
@C0untFloyd
C0untFloyd committed Jan 8, 2024
1 parent a55b78d commit edff845
Show file tree
Hide file tree
Showing 12 changed files with 325 additions and 217 deletions.
9 changes: 9 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,15 @@ Additional commandline arguments are currently unsupported and settings should b

### Changelog

**08.01.2024** v3.5.0

- Bugfix: wrong access options when creating folders
- New auto rotation of horizontal faces, fixing bad landmark positions (expanded on ![PR 364](https://github.com/C0untFloyd/roop-unleashed/pull/364))
- Simple VR Option for stereo Images/Movies, best used in selected face mode
- Added RestoreFormer Enhancer - https://github.com/wzhouxiff/RestoreFormer
- Bumped up package versions for onnx/Torch etc.


**16.10.2023** v3.3.4

**11.8.2023** v2.7.0
Expand Down
27 changes: 12 additions & 15 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,19 @@
--extra-index-url https://download.pytorch.org/whl/cu118

numpy==1.24.2
gradio==3.44.2
opencv-python==4.8.0.76
onnx==1.14.1
numpy==1.26.2
gradio==3.50.2
opencv-python==4.8.1.78
onnx==1.15.0
insightface==0.7.3
psutil==5.9.5
pillow==10.0.1
torch==2.0.1+cu118; sys_platform != 'darwin'
torch==2.0.1; sys_platform == 'darwin'
torchvision==0.15.2+cu118; sys_platform != 'darwin'
torchvision==0.15.2; sys_platform == 'darwin'
onnxruntime==1.16.0; sys_platform == 'darwin' and platform_machine != 'arm64'
psutil==5.9.6
torch==2.1.2+cu118; sys_platform != 'darwin'
torch==2.1.2; sys_platform == 'darwin'
torchvision==0.16.2+cu118; sys_platform != 'darwin'
torchvision==0.16.2; sys_platform == 'darwin'
onnxruntime==1.16.3; sys_platform == 'darwin' and platform_machine != 'arm64'
onnxruntime-silicon==1.13.1; sys_platform == 'darwin' and platform_machine == 'arm64'
onnxruntime-gpu==1.16.1; sys_platform != 'darwin'
protobuf==4.23.2
onnxruntime-gpu==1.16.3; sys_platform != 'darwin'
tqdm==4.66.1
ftfy
regex
pyvirtualcam
imutils==0.5.4
pyvirtualcam
203 changes: 72 additions & 131 deletions roop/ProcessMgr.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@

from roop.ProcessOptions import ProcessOptions

from roop.face_util import get_first_face, get_all_faces, rotate_image_180, rotate_image_90, rotate_anticlockwise, rotate_clockwise
from roop.face_util import get_first_face, get_all_faces, rotate_image_180, rotate_anticlockwise, rotate_clockwise, clamp_cut_values
from roop.utilities import compute_cosine_distance, get_device, str_to_class
import roop.vr_util as vr

from typing import Any, List, Callable
from roop.typing import Frame
from roop.typing import Frame, Face
from concurrent.futures import ThreadPoolExecutor, as_completed
from threading import Thread, Lock
from queue import Queue
Expand Down Expand Up @@ -59,12 +60,13 @@ class ProcessMgr():


plugins = {
'faceswap' : 'FaceSwapInsightFace',
'mask_clip2seg' : 'Mask_Clip2Seg',
'codeformer' : 'Enhance_CodeFormer',
'gfpgan' : 'Enhance_GFPGAN',
'dmdnet' : 'Enhance_DMDNet',
'gpen' : 'Enhance_GPEN',
'faceswap' : 'FaceSwapInsightFace',
'mask_clip2seg' : 'Mask_Clip2Seg',
'codeformer' : 'Enhance_CodeFormer',
'gfpgan' : 'Enhance_GFPGAN',
'dmdnet' : 'Enhance_DMDNet',
'gpen' : 'Enhance_GPEN',
'restoreformer' : 'Enhance_RestoreFormer',
}

def __init__(self, progress):
Expand Down Expand Up @@ -310,40 +312,6 @@ def swap_faces(self, frame, temp_frame):
faces = get_all_faces(frame)
if faces is None:
return num_faces_found, frame

if self.options.swap_mode == "single_face_frames_only":
if len(faces) == 1:
num_faces_found += 1
target_face = faces[0]

temp_frame = self.process_face(self.options.selected_index, target_face, temp_frame)

input_face = self.input_face_datas[self.options.selected_index].faces[0]
rotation_action = self.rotation_action(target_face, frame)
swapped_face = None
optimal_frame = temp_frame.copy()

# before we try and get the swapped face again, we need to make sure we're getting it from the most optimal version of the frame
# otherwise it sometimes doesn't detect it, so if it needs to be rotated, then do that first.
if rotation_action == "rotate_clockwise":
optimal_frame = rotate_clockwise(optimal_frame)
elif rotation_action == "rotate_anticlockwise":
optimal_frame = rotate_anticlockwise(optimal_frame)

swapped_face = get_first_face(optimal_frame)

if swapped_face is None:
num_faces_found = 0
return num_faces_found, frame
else:
# check if the face matches closely the face we intended to swap it too
# if it doesn't, it's probably insightface failing and returning some garbled mess, so skip it
cosine_distance = compute_cosine_distance(swapped_face.embedding, input_face.embedding)
if cosine_distance >= self.options.face_distance_threshold:
num_faces_found = 0
return num_faces_found, frame
else:
return num_faces_found, frame

if self.options.swap_mode == "all":
for face in faces:
Expand All @@ -358,7 +326,8 @@ def swap_faces(self, frame, temp_frame):
if i < len(self.input_face_datas):
temp_frame = self.process_face(i, face, temp_frame)
num_faces_found += 1
break
if not roop.globals.vr_mode:
break
del face
elif self.options.swap_mode == "all_female" or self.options.swap_mode == "all_male":
gender = 'F' if self.options.swap_mode == "all_female" else 'M'
Expand All @@ -368,6 +337,10 @@ def swap_faces(self, frame, temp_frame):
temp_frame = self.process_face(self.options.selected_index, face, temp_frame)
del face

if roop.globals.vr_mode and num_faces_found % 2 > 0:
# stereo image, there has to be an even number of faces
num_faces_found = 0
return num_faces_found, frame
if num_faces_found == 0:
return num_faces_found, frame

Expand All @@ -377,7 +350,7 @@ def swap_faces(self, frame, temp_frame):
return num_faces_found, temp_frame


def rotation_action(self, original_face, frame:Frame):
def rotation_action(self, original_face:Face, frame:Frame):
(height, width) = frame.shape[:2]

bounding_box_width = original_face.bbox[2] - original_face.bbox[0]
Expand Down Expand Up @@ -412,7 +385,7 @@ def rotation_action(self, original_face, frame:Frame):
# this is someone lying down with their face in the left hand side of the frame
return "rotate_clockwise"

return "noop"
return None


def auto_rotate_frame(self, original_face, frame:Frame):
Expand All @@ -423,23 +396,13 @@ def auto_rotate_frame(self, original_face, frame:Frame):

if rotation_action == "rotate_anticlockwise":
#face is horizontal, rotating frame anti-clockwise and getting face bounding box from rotated frame
rotated_bbox = self.rotate_bbox_anticlockwise(original_face.bbox, frame)
frame = rotate_anticlockwise(frame)
target_face = self.get_rotated_target_face(rotated_bbox, frame)
elif rotation_action == "rotate_clockwise":
#face is horizontal, rotating frame clockwise and getting face bounding box from rotated frame
rotated_bbox = self.rotate_bbox_clockwise(original_face.bbox, frame)
frame = rotate_clockwise(frame)
target_face = self.get_rotated_target_face(rotated_bbox, frame)

if target_face is None:
#no face was detected in the rotated frame, so use the original frame and face
target_face = original_face
frame = original_frame
rotation_action = "noop"

return target_face, frame, rotation_action


def auto_unrotate_frame(self, frame:Frame, rotation_action):
if rotation_action == "rotate_anticlockwise":
Expand All @@ -450,90 +413,56 @@ def auto_unrotate_frame(self, frame:Frame, rotation_action):
return frame


def get_rotated_target_face(self, rotated_bbox, rotated_frame:Frame):
rotated_faces = get_all_faces(rotated_frame)

if not rotated_faces:
return None

rotated_target_face = rotated_faces[0]
best_iou = 0

for rotated_face in rotated_faces:
iou = self.intersection_over_union(rotated_bbox, rotated_face.bbox)
if iou > best_iou:
rotated_target_face = rotated_face
best_iou = iou

return rotated_target_face

def process_face(self,face_index, target_face:Face, frame:Frame):
enhanced_frame = None
inputface = self.input_face_datas[face_index].faces[0]

def rotate_bbox_clockwise(self, bbox, frame:Frame):
(height, width) = frame.shape[:2]
rotation_action = None
if roop.globals.autorotate_faces:
# check for sideways rotation of face
rotation_action = self.rotation_action(target_face, frame)
if rotation_action is not None:
(startX, startY, endX, endY) = target_face["bbox"].astype("int")
width = endX - startX
height = endY - startY
offs = int(max(width,height) * 0.25)
rotcutframe,startX, startY, endX, endY = self.cutout(frame, startX - offs, startY - offs, endX + offs, endY + offs)
if rotation_action == "rotate_anticlockwise":
rotcutframe = rotate_anticlockwise(rotcutframe)
elif rotation_action == "rotate_clockwise":
rotcutframe = rotate_clockwise(rotcutframe)
# rotate image and re-detect face to correct wonky landmarks
rotface = get_first_face(rotcutframe)
if rotface is None:
rotation_action = None
else:
saved_frame = frame.copy()
frame = rotcutframe
target_face = rotface

start_x = bbox[0]
start_y = bbox[1]
end_x = bbox[2]
end_y = bbox[3]

#bottom left corner becomes top left corner
#top right corner becomes bottom right corner

rotated_start_x = height - end_y
rotated_start_y = start_x
rotated_end_x = height - start_y
rotated_end_y = end_x
# if roop.globals.vr_mode:
# bbox = target_face.bbox
# [orig_width, orig_height, _] = frame.shape

return [rotated_start_x, rotated_start_y, rotated_end_x, rotated_end_y]
# # Convert bounding box to ints
# x1, y1, x2, y2 = map(int, bbox)

# # Determine the center of the bounding box
# x_center = (x1 + x2) / 2
# y_center = (y1 + y2) / 2

def rotate_bbox_anticlockwise(self, bbox, frame:Frame):

(height, width) = frame.shape[:2]
# # Normalize coordinates to range [-1, 1]
# x_center_normalized = x_center / (orig_width / 2) - 1
# y_center_normalized = y_center / (orig_width / 2) - 1

start_x = bbox[0]
start_y = bbox[1]
end_x = bbox[2]
end_y = bbox[3]

# So the algorithm is
# - top right corner translates to top left corner which gives start_x, start_y and is calculated as follows: (start_y, width - end_x)
# - bottom left corner translates to bottom right corner giving end_x, end_y and is calculated as follows: (end_y, width - start_x)

rotated_start_x = start_y
rotated_start_y = width - end_x
rotated_end_x = end_y
rotated_end_y = width - start_x

return [rotated_start_x, rotated_start_y, rotated_end_x, rotated_end_y]


def intersection_over_union(self,boxA, boxB):
# https://pyimagesearch.com/2016/11/07/intersection-over-union-iou-for-object-detection/
# determine the (x, y)-coordinates of the intersection rectangle
xA = max(boxA[0], boxB[0])
yA = max(boxA[1], boxB[1])
xB = min(boxA[2], boxB[2])
yB = min(boxA[3], boxB[3])
# compute the area of intersection rectangle
interArea = max(0, xB - xA + 1) * max(0, yB - yA + 1)
# compute the area of both the prediction and ground-truth
# rectangles
boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1)
boxBArea = (boxB[2] - boxB[0] + 1) * (boxB[3] - boxB[1] + 1)
# compute the intersection over union by taking the intersection
# area and dividing it by the sum of prediction + ground-truth
# areas - the interesection area
iou = interArea / float(boxAArea + boxBArea - interArea)
# return the intersection over union value
return iou


def process_face(self,face_index, target_face, frame:Frame):
target_face, frame, rotation_action = self.auto_rotate_frame(target_face, frame)
# # Convert normalized coordinates to spherical (theta, phi)
# theta = x_center_normalized * 180 # Theta ranges from -180 to 180 degrees
# phi = -y_center_normalized * 90 # Phi ranges from -90 to 90 degrees

enhanced_frame = None
inputface = self.input_face_datas[face_index].faces[0]
# img = vr.GetPerspective(frame, 90, theta, phi, 1280, 1280) # Generate perspective image

for p in self.processors:
if p.type == 'swap':
Expand All @@ -546,6 +475,7 @@ def process_face(self,face_index, target_face, frame:Frame):

upscale = 512
orig_width = fake_frame.shape[1]

fake_frame = cv2.resize(fake_frame, (upscale, upscale), cv2.INTER_CUBIC)
mask_offsets = inputface.mask_offsets

Expand All @@ -555,7 +485,11 @@ def process_face(self,face_index, target_face, frame:Frame):
else:
result = self.paste_upscale(fake_frame, enhanced_frame, target_face.matrix, frame, scale_factor, mask_offsets)

return self.auto_unrotate_frame(result, rotation_action)
if rotation_action is not None:
fake_frame = self.auto_unrotate_frame(result, rotation_action)
return self.paste_simple(fake_frame, saved_frame, startX, startY)

return result



Expand All @@ -571,6 +505,13 @@ def cutout(self, frame:Frame, start_x, start_y, end_x, end_y):
end_y = frame.shape[0]
return frame[start_y:end_y, start_x:end_x], start_x, start_y, end_x, end_y

def paste_simple(self, src:Frame, dest:Frame, start_x, start_y):
end_x = start_x + src.shape[1]
end_y = start_y + src.shape[0]

start_x, end_x, start_y, end_y = clamp_cut_values(start_x, end_x, start_y, end_y, dest)
dest[start_y:end_y, start_x:end_x] = src
return dest


# Paste back adapted from here
Expand Down
4 changes: 3 additions & 1 deletion roop/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def pre_check() -> bool:
util.conditional_download(download_directory_path, ['https://huggingface.co/countfloyd/deepfake/resolve/main/GFPGANv1.4.onnx'])
util.conditional_download(download_directory_path, ['https://github.com/csxmli2016/DMDNet/releases/download/v1/DMDNet.pth'])
util.conditional_download(download_directory_path, ['https://github.com/facefusion/facefusion-assets/releases/download/models/GPEN-BFR-512.onnx'])

util.conditional_download(download_directory_path, ['https://huggingface.co/countfloyd/deepfake/resolve/main/restoreformer.onnx'])
download_directory_path = util.resolve_relative_path('../models/CLIP')
util.conditional_download(download_directory_path, ['https://huggingface.co/countfloyd/deepfake/resolve/main/rd64-uni-refined.pth'])
download_directory_path = util.resolve_relative_path('../models/CodeFormer')
Expand Down Expand Up @@ -173,6 +173,8 @@ def get_processing_plugins(use_clip):
processors += ",dmdnet"
elif roop.globals.selected_enhancer == 'GPEN':
processors += ",gpen"
elif roop.globals.selected_enhancer == 'Restoreformer':
processors += ",restoreformer"
return processors


Expand Down
Loading

0 comments on commit edff845

Please sign in to comment.