1. OpenPose
인간 자세 예측의 한 분야로, 오로지 카메라 한 대로만 가지고 사람의 몸, 얼굴, 손가락 마디를 정확하게 예측 하는 것
신체의 관절을 추론 후, 관절들을 이어주는 방식
- BODY-25
- 출력 관절이 25개
- COCO
- 출력 관절이 18개
- MPII
- 출력 관절이 15개
=> 달리기 자세를 측정해야 하기에 많은 관절을 추출할 수 있는 BODY-25를 사용하기로 했다
2. OpenPose 간단히 실습해보기
OpenPose를 사용하기 위해서는 깃허브에서 모델을 다운받아서 적용해야 한다
방법
- OpenPose 저장소를 clone한다 (https://github.com/CMU-Perceptual-Computing-Lab/openpose)
- models 폴더 안의 getModels.sh를 실행한다
- 생성된 파일 중 models/pose안의 모델을 하나 선택하여 폴더 안에 생성된 prototxt 파일과 caffemodel 파일을 원하는 위치에 복사한다
코드 작성
import cv2
# MPII에서 각 파트 번호, 선으로 연결될 POSE_PAIRS
BODY_PARTS = { "Head": 0, "Neck": 1, "RShoulder": 2, "RElbow": 3, "RWrist": 4,
"LShoulder": 5, "LElbow": 6, "LWrist": 7, "RHip": 8, "RKnee": 9,
"RAnkle": 10, "LHip": 11, "LKnee": 12, "LAnkle": 13, "Chest": 14,
"Background": 15 }
POSE_PAIRS = [ ["Head", "Neck"], ["Neck", "RShoulder"], ["RShoulder", "RElbow"],
["RElbow", "RWrist"], ["Neck", "LShoulder"], ["LShoulder", "LElbow"],
["LElbow", "LWrist"], ["Neck", "Chest"], ["Chest", "RHip"], ["RHip", "RKnee"],
["RKnee", "RAnkle"], ["Chest", "LHip"], ["LHip", "LKnee"], ["LKnee", "LAnkle"] ]
# 각 파일 path
protoFile = "pose_deploy_linevec_faster_4_stages.prototxt"
weightsFile = "pose_iter_160000.caffemodel"
# 위의 path에 있는 network 불러오기
net = cv2.dnn.readNetFromCaffe(protoFile, weightsFile)
# 이미지 읽어오기
image = cv2.imread("image2.png")
# frame.shape = 불러온 이미지에서 height, width, color 받아옴
imageHeight, imageWidth, _ = image.shape
# network에 넣기위해 전처리
inpBlob = cv2.dnn.blobFromImage(image, 1.0 / 255, (imageWidth, imageHeight), (0, 0, 0), swapRB=False, crop=False)
# network에 넣어주기
net.setInput(inpBlob)
# 결과 받아오기
output = net.forward()
# output.shape[0] = 이미지 ID, [1] = 출력 맵의 높이, [2] = 너비
H = output.shape[2]
W = output.shape[3]
print("이미지 ID : ", len(output[0]), ", H : ", output.shape[2], ", W : ",output.shape[3]) # 이미지 ID
# 키포인트 검출시 이미지에 그려줌
points = []
for i in range(0,15):
# 해당 신체부위 신뢰도 얻음.
probMap = output[0, i, :, :]
# global 최대값 찾기
minVal, prob, minLoc, point = cv2.minMaxLoc(probMap)
# 원래 이미지에 맞게 점 위치 변경
x = (imageWidth * point[0]) / W
y = (imageHeight * point[1]) / H
# 키포인트 검출한 결과가 0.1보다 크면(검출한곳이 위 BODY_PARTS랑 맞는 부위면) points에 추가, 검출했는데 부위가 없으면 None으로
if prob > 0.1 :
cv2.circle(image, (int(x), int(y)), 3, (0, 255, 255), thickness=-1, lineType=cv2.FILLED) # circle(그릴곳, 원의 중심, 반지름, 색)
cv2.putText(image, "{}".format(i), (int(x), int(y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1, lineType=cv2.LINE_AA)
points.append((int(x), int(y)))
else :
points.append(None)
cv2.imshow("Output-Keypoints",image)
cv2.waitKey(0)
# 이미지 복사
imageCopy = image
# 각 POSE_PAIRS별로 선 그어줌 (머리 - 목, 목 - 왼쪽어깨, ...)
for pair in POSE_PAIRS:
partA = pair[0] # Head
partA = BODY_PARTS[partA] # 0
partB = pair[1] # Neck
partB = BODY_PARTS[partB] # 1
#print(partA," 와 ", partB, " 연결\n")
if points[partA] and points[partB]:
cv2.line(imageCopy, points[partA], points[partB], (0, 255, 0), 2)
cv2.imshow("Output-Keypoints",imageCopy)
cv2.waitKey(0)
cv2.destroyAllWindows()
참고한 블로그
이렇게 간단히 OpenPose를 사용하는 법을 알아보았다
MPII를 사용한 코드를 가지고 왔지만, MPII뿐만 아니라 BODY25를 사용하여 검출하는데 성공했다
하지만 이렇게만 하면 검출이 되긴 하지만, 제대로 안될 것 이다
BODY25를 사용했을때, treadmill의 일정 부근을 발 끝으로 인식하는 등 제대로 검출을 하지 못하는 모습을 보였다
3. BODY-25를 사용해 제대로 검출하기
검출이 제대로 되지 않은 모습을 보며, 막막했던 찰나 팀원분이 블로그를 하나 알려주셨다
블로그 작성하신 분 정말 감사합니다
import cv2
import time
import numpy as np
import matplotlib.pyplot as plt
weightsFile = 'C:/Users/Admin/Downloads/openpose-1.7.0-binaries-win64-gpu-python3.7-flir-3d_recommended/openpose/models/pose/body_25/pose_iter_584000.caffemodel'
protoFile = 'C:/Users/Admin/Downloads/openpose-1.7.0-binaries-win64-gpu-python3.7-flir-3d_recommended/openpose/models/pose/body_25/pose_deploy.prototxt'
nPoints = 25
keypointsMapping = ["Nose","Neck","RShoulder", "RElbow", "RWrist", "LShoulder", "LElbow",
"LWrist", "MidHip", "RHip","RKnee", "RAnkle", "LHip", "LKnee",
"LAnkle", "REye", "LEye", "REar", "LEar", "LBigToe", "LSmallToe",
"LHeel", "RBigToe", "RSmallToe", "RHeel"]
POSE_PAIRS = [[1,2], [1,5], [2,3], [3,4], [5,6], [6,7],
[1,8], [8,9], [9,10], [10,11], [8,12], [12,13], [13,14],
[11,24], [11,22], [22,23], [14,21],[14,19],[19,20],
[1,0], [0,15], [15,17], [0,16], [16,18],
[2,17], [5,18]]
mapIdx = [[40,41],[48,49],[42,43],[44,45],[50,51],[52,53],
[26,27],[32,33],[28,29],[30,31],[34,35],[36,37],
[38,39],[76,77],[72,73],[74,75],[70,71],[66,67],
[68,69],[56,57],[58,59],[62,63],[60,61],[64,65],
[46,47],[54,55]]
colors = [ [0,100,255], [0,100,255], [0,255,255], [0,100,255], [0,255,255], [0,100,255],
[0,255,0], [255,200,100], [255,0,255], [0,255,0], [255,200,100], [255,0,255],
[0,0,255], [255,0,0], [200,200,0], [255,0,0], [125,200,125], [125,200,0],
[200,200,200],[200,100,200],[200,200,0],[0,200,0],[200,0,255],[0,250,125],
[0,200,0],[0,120,200]]
device = 'gpu'
def getKeypoints(probMap, threshold=0.1):
mapSmooth = cv2.GaussianBlur(probMap,(3,3),0,0)
mapMask = np.uint8(mapSmooth>threshold)
keypoints = []
#find the blobs
contours, _ = cv2.findContours(mapMask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
#for each blob find the maxima
for cnt in contours:
blobMask = np.zeros(mapMask.shape)
blobMask = cv2.fillConvexPoly(blobMask, cnt, 1)
maskedProbMap = mapSmooth * blobMask
#maxVal = 있을 확률
#maxLoc = x,y좌표
_, maxVal, _, maxLoc = cv2.minMaxLoc(maskedProbMap)
keypoints.append(maxLoc + (probMap[maxLoc[1], maxLoc[0]],))
return keypoints
def getValidPairs(output):
valid_pairs = []
invalid_pairs = []
n_interp_samples = 10 # 벡터를 나눌 수
paf_score_th = 0.1
conf_th = 0.7
# loop for every POSE_PAIR
for k in range(len(mapIdx)):
# A->B constitute a limb
pafA = output[0, mapIdx[k][0], :, :]
pafB = output[0, mapIdx[k][1], :, :]
pafA = cv2.resize(pafA, (frameWidth, frameHeight))
pafB = cv2.resize(pafB, (frameWidth, frameHeight))
# Find the keypoints for the first and second limb
candA = detected_keypoints[POSE_PAIRS[k][0]]
candB = detected_keypoints[POSE_PAIRS[k][1]]
nA = len(candA)
nB = len(candB)
# If keypoints for the joint-pair is detected
# check every joint in candA with every joint in candB
# Calculate the distance vector between the two joints
# Find the PAF values at a set of interpolated points between the joints
# Use the above formula to compute a score to mark the connection valid
if( nA != 0 and nB != 0):
valid_pair = np.zeros((0,3))
for i in range(nA):
max_j=-1
maxScore = -1
found = 0
for j in range(nB):
# Find d_ij
d_ij = np.subtract(candB[j][:2], candA[i][:2])
norm = np.linalg.norm(d_ij)
if norm:
d_ij = d_ij / norm
else:
continue
# Find p(u)
interp_coord = list(zip(np.linspace(candA[i][0], candB[j][0], num=n_interp_samples),
np.linspace(candA[i][1], candB[j][1], num=n_interp_samples)))
# Find L(p(u))
paf_interp = []
for k in range(len(interp_coord)):
paf_interp.append([pafA[int(round(interp_coord[k][1])), int(round(interp_coord[k][0]))],
pafB[int(round(interp_coord[k][1])), int(round(interp_coord[k][0]))] ])
# Find E
paf_scores = np.dot(paf_interp, d_ij)
avg_paf_score = sum(paf_scores)/len(paf_scores)
# Check if the connection is valid
# If the fraction of interpolated vectors aligned with PAF is higher then threshold -> Valid Pair
if ( len(np.where(paf_scores > paf_score_th)[0]) / n_interp_samples ) > conf_th :
if avg_paf_score > maxScore:
max_j = j
maxScore = avg_paf_score
found = 1
# Append the connection to the list
if found:
valid_pair = np.append(valid_pair, [[candA[i][3], candB[max_j][3], maxScore]], axis=0)
# Append the detected connections to the global list
valid_pairs.append(valid_pair)
else: # If no keypoints are detected
print("No Connection : k = {}".format(k))
invalid_pairs.append(k)
valid_pairs.append([])
print(valid_pairs)
return valid_pairs, invalid_pairs
# This function creates a list of keypoints belonging to each person
# For each detected valid pair, it assigns the joint(s) to a person
# It finds the person and index at which the joint should be added. This can be done since we have an id for each joint
def getPersonwiseKeypoints(valid_pairs, invalid_pairs):
# the last number in each row is the overall score
personwiseKeypoints = -1 * np.ones((0, 26))
for k in range(len(mapIdx)):
if k not in invalid_pairs:
partAs = valid_pairs[k][:,0]
partBs = valid_pairs[k][:,1]
indexA, indexB = np.array(POSE_PAIRS[k])
for i in range(len(valid_pairs[k])):
found = 0
person_idx = -1
for j in range(len(personwiseKeypoints)):
if personwiseKeypoints[j][indexA] == partAs[i]:
person_idx = j
found = 1
break
if found:
personwiseKeypoints[person_idx][indexB] = partBs[i]
personwiseKeypoints[person_idx][-1] += keypoints_list[partBs[i].astype(int), 2] + valid_pairs[k][i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 24:
row = -1 * np.ones(26)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
# add the keypoint_scores for the two keypoints and the paf_score
row[-1] = sum(keypoints_list[valid_pairs[k][i,:2].astype(int), 2]) + valid_pairs[k][i][2]
personwiseKeypoints = np.vstack([personwiseKeypoints, row])
return personwiseKeypoints
image1 = cv2.imread('C:/Users/Admin/Downloads/openpose-1.7.0-binaries-win64-gpu-python3.7-flir-3d_recommended/openpose/examples/lala/lala4.jpg')
frameWidth = image1.shape[1]
frameHeight = image1.shape[0]
t = time.time()
net = cv2.dnn.readNetFromCaffe(protoFile, weightsFile)
if device == "cpu":
net.setPreferableBackend(cv2.dnn.DNN_TARGET_CPU)
print("Using CPU device")
elif device == "gpu":
net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA)
net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA)
print("Using GPU device")
# Fix the input Height and get the width according to the Aspect Ratio
inHeight = 368
inWidth = int((inHeight/frameHeight)*frameWidth)
inpBlob = cv2.dnn.blobFromImage(image1,
1.0 / 255, (inWidth, inHeight),
(0, 0, 0), swapRB=False, crop=False)
net.setInput(inpBlob)
output = net.forward()
print("Time Taken = {}".format(time.time() - t))
for i in range(78):
n = i+1
probMap = output[0,i, :, :]
probMap = cv2.resize(probMap, (frameWidth, frameHeight))
plt.figure(figsize = (7,5))
plt.imshow(cv2.cvtColor(image1, cv2.COLOR_BGR2RGB))
plt.imshow(probMap, alpha=0.6)
plt.colorbar()
plt.axis("off")
plt.show()
detected_keypoints = []
keypoints_list = np.zeros((0,3))
keypoint_id = 0
threshold = 0.1
for part in range(nPoints):
probMap = output[0,part,:,:]
probMap = cv2.resize(probMap, (image1.shape[1], image1.shape[0]))
# plt.figure()
# plt.imshow(255*np.uint8(probMap>threshold))
keypoints = getKeypoints(probMap, threshold)
print("Keypoints - {} : {}".format(keypointsMapping[part], keypoints))
keypoints_with_id = []
for i in range(len(keypoints)):
keypoints_with_id.append(keypoints[i] + (keypoint_id,))
keypoints_list = np.vstack([keypoints_list, keypoints[i]])
keypoint_id += 1
detected_keypoints.append(keypoints_with_id)
print(detected_keypoints)
frameClone = image1.copy()
for i in range(nPoints):
for j in range(len(detected_keypoints[i])):
cv2.circle(frameClone, detected_keypoints[i][j][0:2], 3, [0,0,255], -1, cv2.LINE_AA)
plt.figure(figsize=[15,15])
plt.imshow(frameClone[:,:,[2,1,0]])
valid_pairs, invalid_pairs = getValidPairs(output)
personwiseKeypoints = getPersonwiseKeypoints(valid_pairs, invalid_pairs)
for i in range(24):
for n in range(len(personwiseKeypoints)):
index = personwiseKeypoints[n][np.array(POSE_PAIRS[i])]
if -1 in index:
continue
B = np.int32(keypoints_list[index.astype(int), 0])
A = np.int32(keypoints_list[index.astype(int), 1])
cv2.line(frameClone, (B[0], A[0]), (B[1], A[1]), colors[i], 3, cv2.LINE_AA)
plt.figure(figsize=[10,8])
plt.imshow(frameClone[:,:,[2,1,0]])
참고한 블로그
위의 블로그의 코드를 통해 정확하게 Keypoints를 검출할 수 있었다
'Project > 졸업프로젝트 - Running Coach' 카테고리의 다른 글
| [졸업 프로젝트] #1 주제 정하기 (0) | 2023.04.28 |
|---|