sa/HPE_mediapipe
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

First commit

Browse Source
This commit is contained in:
Arkadiy Strelnikov
2021-11-26 01:47:03 +07:00
commit 5459e0f0ce
6 changed files with 1054 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
.gitignore vendored Normal file
View File

@@ -0,0 +1,134 @@
# ---> Python
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
datacndm
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
.idea*
*.log

479
ModelUE4.py Normal file
View File

@@ -0,0 +1,479 @@
# convert mediapipe model to model from Unreal
# element of json
def makexyzrotvis(x,y,z,rotx,roty,rotz,visible):
ret = {}
translation = {}
rotation = {}
scale = {}
translation["x"] = x
translation["y"] = y
translation["z"] = z
rotation["rotx"] = rotx
rotation["roty"] = roty
rotation["rotz"] = rotz
scale["x"] = 1.0
scale["y"] = 1.0
scale["z"] = 1.0
ret["visible"] = visible
ret["translation"] = translation
ret["rotation"] = rotation
ret["scale"] = scale
return ret
def bodyconvert(poseslms, data, coef, maxy):
dataxyz = {}
rootx = poseslms[24].x * coef
rooty = -poseslms[24].z * coef
rootz = -poseslms[24].y * coef
rootv = poseslms[24].visibility
data["hip_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["hip_r"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[23].x * coef
rooty = -poseslms[23].z * coef
rootz = -poseslms[23].y * coef
rootv = poseslms[23].visibility
data["hip_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["hip_l"] = [rootx, rooty, rootz, rootv]
rootx = 0.5 * (dataxyz["hip_l"][0] + dataxyz["hip_r"][0]) * coef
rooty = -maxy * coef
rootz = -0.5 * (dataxyz["hip_l"][2] + dataxyz["hip_r"][2]) * coef
rootv = 1.0
data["root"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["root"] = [rootx, rooty, rootz, rootv]
rootx = 0.5 * (dataxyz["hip_l"][0] + dataxyz["hip_r"][0]) * coef
rooty = -0.5 * (dataxyz["hip_l"][1] + dataxyz["hip_r"][1]) * coef
rootz = -0.5 * (dataxyz["hip_l"][2] + dataxyz["hip_r"][2]) * coef
rootv = 0.5 * (dataxyz["hip_l"][3] + dataxyz["hip_r"][3])
data["pelvis"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["pelvis"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[0].x * coef
rooty = -poseslms[0].z * coef
rootz = -poseslms[0].y * coef
rootv = poseslms[0].visibility
data["head"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["head"] = [rootx, rooty, rootz, rootv]
rootx = (0.955887 * dataxyz["head"][0] + 0.044113 * dataxyz["pelvis"][0]) * coef
rooty = (0.955887 * dataxyz["head"][1] + 0.044113 * dataxyz["pelvis"][1]) * coef
rootz = (0.955887 * dataxyz["head"][2] + 0.044113 * dataxyz["pelvis"][2]) * coef
rootv = (0.955887 * dataxyz["head"][3] + 0.044113 * dataxyz["pelvis"][3])
data["spine_01"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["spine_01"] = [rootx, rooty, rootz, rootv]
rootx = (0.864201 * dataxyz["head"][0] + 0.135799 * dataxyz["pelvis"][0]) * coef
rooty = (0.864201 * dataxyz["head"][1] + 0.135799 * dataxyz["pelvis"][1]) * coef
rootz = (0.864201 * dataxyz["head"][2] + 0.135799 * dataxyz["pelvis"][2]) * coef
rootv = (0.864201 * dataxyz["head"][3] + 0.135799 * dataxyz["pelvis"][3])
data["spine_02"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["spine_02"] = [rootx, rooty, rootz, rootv]
rootx = (0.717787 * dataxyz["head"][0] + 0.282213 * dataxyz["pelvis"][0]) * coef
rooty = (0.717787 * dataxyz["head"][1] + 0.282213 * dataxyz["pelvis"][1]) * coef
rootz = (0.717787 * dataxyz["head"][2] + 0.282213 * dataxyz["pelvis"][2]) * coef
rootv = (0.717787 * dataxyz["head"][3] + 0.282213 * dataxyz["pelvis"][3])
data["spine_03"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["spine_03"] = [rootx, rooty, rootz, rootv]
rootx = (0.548406 * dataxyz["head"][0] + 0.451594 * dataxyz["pelvis"][0]) * coef
rooty = (0.548406 * dataxyz["head"][1] + 0.451594 * dataxyz["pelvis"][1]) * coef
rootz = (0.548406 * dataxyz["head"][2] + 0.451594 * dataxyz["pelvis"][2]) * coef
rootv = (0.548406 * dataxyz["head"][3] + 0.451594 * dataxyz["pelvis"][3])
data["spine_04"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["spine_04"] = [rootx, rooty, rootz, rootv]
rootx = (0.228975 * dataxyz["head"][0] + 0.771025 * dataxyz["pelvis"][0]) * coef
rooty = (0.228975 * dataxyz["head"][1] + 0.771025 * dataxyz["pelvis"][1]) * coef
rootz = (0.228975 * dataxyz["head"][2] + 0.771025 * dataxyz["pelvis"][2]) * coef
rootv = (0.228975 * dataxyz["head"][3] + 0.771025 * dataxyz["pelvis"][3])
data["spine_05"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["spine_05"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[26].x * coef
rooty = -poseslms[26].z * coef
rootz = -poseslms[26].y * coef
rootv = poseslms[26].visibility
data["knee_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["knee_r"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[25].x * coef
rooty = -poseslms[25].z * coef
rootz = -poseslms[25].y * coef
rootv = poseslms[25].visibility
data["knee_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["knee_l"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[28].x * coef
rooty = -poseslms[28].z * coef
rootz = -poseslms[28].y * coef
rootv = poseslms[28].visibility
data["ankle_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["ankle_r"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[27].x * coef
rooty = -poseslms[27].z * coef
rootz = -poseslms[27].y * coef
rootv = poseslms[27].visibility
data["ankle_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["ankle_l"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[32].x * coef
rooty = -poseslms[32].z * coef
rootz = -poseslms[32].y * coef
rootv = poseslms[32].visibility
data["ball_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["ball_r"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[31].x * coef
rooty = -poseslms[31].z * coef
rootz = -poseslms[31].y * coef
rootv = poseslms[31].visibility
data["ball_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["ball_l"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[30].x * coef
rooty = -poseslms[30].z * coef
rootz = -poseslms[30].y * coef
rootv = poseslms[30].visibility
data["heel_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["heel_r"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[29].x * coef
rooty = -poseslms[29].z * coef
rootz = -poseslms[29].y * coef
rootv = poseslms[29].visibility
data["heel_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["heel_l"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[12].x * coef
rooty = -poseslms[12].z * coef
rootz = -poseslms[12].y * coef
rootv = poseslms[12].visibility
data["shoulder_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["shoulder_r"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[11].x * coef
rooty = -poseslms[11].z * coef
rootz = -poseslms[11].y * coef
rootv = poseslms[11].visibility
data["shoulder_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["shoulder_l"] = [rootx, rooty, rootz, rootv]
rootx = (0.635 * dataxyz["shoulder_r"][0] + 0.365 * dataxyz["shoulder_l"][0]) * coef
rooty = 0.5 * (0.635 * dataxyz["shoulder_r"][1] + 0.365 * dataxyz["shoulder_l"][1] + 0.31 * dataxyz["head"][1] + 0.69 * dataxyz["spine_05"][1]) * coef
rootz = (0.0635 * dataxyz["shoulder_r"][1] + 0.0365 * dataxyz["shoulder_l"][1] + 0.9 * dataxyz["spine_05"][1]) * coef
rootv = (0.635 * dataxyz["shoulder_r"][0] + 0.451594 * dataxyz["shoulder_l"][0])
data["clavicle_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["clavicle_r"] = [rootx, rooty, rootz, rootv]
rootx = (0.635 * dataxyz["shoulder_l"][0] + 0.365 * dataxyz["shoulder_r"][0]) * coef
rooty = 0.5 * (0.635 * dataxyz["shoulder_l"][1] + 0.365 * dataxyz["shoulder_r"][1] + 0.31 * dataxyz["head"][1] + 0.69 * dataxyz["spine_05"][1]) * coef
rootz = (0.0635 * dataxyz["shoulder_l"][1] + 0.0365 * dataxyz["shoulder_r"][1] + 0.9 * dataxyz["spine_05"][1]) * coef
rootv = (0.635 * dataxyz["shoulder_l"][0] + 0.365 * dataxyz["shoulder_r"][0])
data["clavicle_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["clavicle_l"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[14].x * coef
rooty = -poseslms[14].z * coef
rootz = -poseslms[14].y * coef
rootv = poseslms[14].visibility
data["elbow_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["elbow_r"] = [rootx, rooty, rootz, rootv]
rootx = poseslms[13].x * coef
rooty = -poseslms[13].z * coef
rootz = -poseslms[13].y * coef
rootv = poseslms[13].visibility
data["elbow_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["elbow_l"] = [rootx, rooty, rootz, rootv]
def rhandconvert(rhandlms, data, coef):
dataxyz = {}
rootx = rhandlms[0].x * coef
rooty = -rhandlms[0].z * coef
rootz = -rhandlms[0].y * coef
rootv = 1.0
data["hand_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["hand_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[5].x * coef
rooty = -rhandlms[5].z * coef
rootz = -rhandlms[5].y * coef
rootv = 1.0
data["index_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["index_2_r"] = [rootx, rooty, rootz, rootv]
rootx = (0.75 * dataxyz["hand_r"][0] + 0.25 * dataxyz["index_2_r"][0]) * coef
rooty = (0.75 * dataxyz["hand_r"][1] + 0.25 * dataxyz["index_2_r"][1]) * coef
rootz = (0.75 * dataxyz["hand_r"][2] + 0.25 * dataxyz["index_2_r"][2]) * coef
rootv = 1.0
data["index_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["index_1_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[9].x * coef
rooty = -rhandlms[9].z * coef
rootz = -rhandlms[9].y * coef
rootv = 1.0
data["middle_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["middle_2_r"] = [rootx, rooty, rootz, rootv]
rootx = (0.75 * dataxyz["hand_r"][0] + 0.25 * dataxyz["middle_2_r"][0]) * coef
rooty = (0.75 * dataxyz["hand_r"][1] + 0.25 * dataxyz["middle_2_r"][1]) * coef
rootz = (0.75 * dataxyz["hand_r"][2] + 0.25 * dataxyz["middle_2_r"][2]) * coef
rootv = 1.0
data["middle_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["middle_1_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[13].x * coef
rooty = -rhandlms[13].z * coef
rootz = -rhandlms[13].y * coef
rootv = 1.0
data["ring_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["ring_2_r"] = [rootx, rooty, rootz, rootv]
rootx = (0.75 * dataxyz["hand_r"][0] + 0.25 * dataxyz["ring_2_r"][0]) * coef
rooty = (0.75 * dataxyz["hand_r"][1] + 0.25 * dataxyz["ring_2_r"][1]) * coef
rootz = (0.75 * dataxyz["hand_r"][2] + 0.25 * dataxyz["ring_2_r"][2]) * coef
rootv = 1.0
data["ring_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["ring_1_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[17].x * coef
rooty = -rhandlms[17].z * coef
rootz = -rhandlms[17].y * coef
rootv = 1.0
data["pinky_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["pinky_2_r"] = [rootx, rooty, rootz, rootv]
rootx = (0.75 * dataxyz["hand_r"][0] + 0.25 * dataxyz["pinky_2_r"][0]) * coef
rooty = (0.75 * dataxyz["hand_r"][1] + 0.25 * dataxyz["pinky_2_r"][1]) * coef
rootz = (0.75 * dataxyz["hand_r"][2] + 0.25 * dataxyz["pinky_2_r"][2]) * coef
rootv = 1.0
data["pinky_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["pinky_1_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[1].x * coef
rooty = -rhandlms[1].z * coef
rootz = -rhandlms[1].y * coef
rootv = 1.0
data["thumb_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["thumb_1_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[2].x * coef
rooty = -rhandlms[2].z * coef
rootz = -rhandlms[2].y * coef
rootv = 1.0
data["thumb_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["thumb_2_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[3].x * coef
rooty = -rhandlms[3].z * coef
rootz = -rhandlms[3].y * coef
rootv = 1.0
data["thumb_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["thumb_3_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[6].x * coef
rooty = -rhandlms[6].z * coef
rootz = -rhandlms[6].y * coef
rootv = 1.0
data["index_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["index_3_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[7].x * coef
rooty = -rhandlms[7].z * coef
rootz = -rhandlms[7].y * coef
rootv = 1.0
data["index_4_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["index_4_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[10].x * coef
rooty = -rhandlms[10].z * coef
rootz = -rhandlms[10].y * coef
rootv = 1.0
data["middle_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["middle_3_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[11].x * coef
rooty = -rhandlms[11].z * coef
rootz = -rhandlms[11].y * coef
rootv = 1.0
data["middle_4_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["middle_4_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[14].x * coef
rooty = -rhandlms[14].z * coef
rootz = -rhandlms[14].y * coef
rootv = 1.0
data["ring_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["ring_3_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[15].x * coef
rooty = -rhandlms[15].z * coef
rootz = -rhandlms[15].y * coef
rootv = 1.0
data["ring_4_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["ring_4_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[18].x * coef
rooty = -rhandlms[18].z * coef
rootz = -rhandlms[18].y * coef
rootv = 1.0
data["pinky_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["pinky_3_r"] = [rootx, rooty, rootz, rootv]
rootx = rhandlms[19].x * coef
rooty = -rhandlms[19].z * coef
rootz = -rhandlms[19].y * coef
rootv = 1.0
data["pinky_4_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["pinky_4_r"] = [rootx, rooty, rootz, rootv]
def lhandconvert(lhandlms, data, coef):
dataxyz = {}
rootx = lhandlms[0].x * coef
rooty = -lhandlms[0].z * coef
rootz = -lhandlms[0].y * coef
rootv = 1.0
data["hand_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["hand_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[5].x * coef
rooty = -lhandlms[5].z * coef
rootz = -lhandlms[5].y * coef
rootv = 1.0
data["index_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["index_2_l"] = [rootx, rooty, rootz, rootv]
rootx = (0.75 * dataxyz["hand_l"][0] + 0.25 * dataxyz["index_2_l"][0]) * coef
rooty = (0.75 * dataxyz["hand_l"][1] + 0.25 * dataxyz["index_2_l"][1]) * coef
rootz = (0.75 * dataxyz["hand_l"][2] + 0.25 * dataxyz["index_2_l"][2]) * coef
rootv = 1.0
data["index_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["index_1_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[9].x * coef
rooty = -lhandlms[9].z * coef
rootz = -lhandlms[9].y * coef
rootv = 1.0
data["middle_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["middle_2_l"] = [rootx, rooty, rootz, rootv]
rootx = (0.75 * dataxyz["hand_l"][0] + 0.25 * dataxyz["middle_2_l"][0]) * coef
rooty = (0.75 * dataxyz["hand_l"][1] + 0.25 * dataxyz["middle_2_l"][1]) * coef
rootz = (0.75 * dataxyz["hand_l"][2] + 0.25 * dataxyz["middle_2_l"][2]) * coef
rootv = 1.0
data["middle_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["middle_1_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[13].x * coef
rooty = -lhandlms[13].z * coef
rootz = -lhandlms[13].y * coef
rootv = 1.0
data["ring_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["ring_2_l"] = [rootx, rooty, rootz, rootv]
rootx = (0.75 * dataxyz["hand_l"][0] + 0.25 * dataxyz["ring_2_l"][0]) * coef
rooty = (0.75 * dataxyz["hand_l"][1] + 0.25 * dataxyz["ring_2_l"][1]) * coef
rootz = (0.75 * dataxyz["hand_l"][2] + 0.25 * dataxyz["ring_2_l"][2]) * coef
rootv = 1.0
data["ring_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["ring_1_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[17].x * coef
rooty = -lhandlms[17].z * coef
rootz = -lhandlms[17].y * coef
rootv = 1.0
data["pinky_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["pinky_2_l"] = [rootx, rooty, rootz, rootv]
rootx = (0.75 * dataxyz["hand_l"][0] + 0.25 * dataxyz["pinky_2_l"][0]) * coef
rooty = (0.75 * dataxyz["hand_l"][1] + 0.25 * dataxyz["pinky_2_l"][1]) * coef
rootz = (0.75 * dataxyz["hand_l"][2] + 0.25 * dataxyz["pinky_2_l"][2]) * coef
rootv = 1.0
data["pinky_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["pinky_1_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[1].x * coef
rooty = -lhandlms[1].z * coef
rootz = -lhandlms[1].y * coef
rootv = 1.0
data["thumb_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["thumb_1_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[2].x * coef
rooty = -lhandlms[2].z * coef
rootz = -lhandlms[2].y * coef
rootv = 1.0
data["thumb_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["thumb_2_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[3].x * coef
rooty = -lhandlms[3].z * coef
rootz = -lhandlms[3].y * coef
rootv = 1.0
data["thumb_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["thumb_3_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[6].x * coef
rooty = -lhandlms[6].z * coef
rootz = -lhandlms[6].y * coef
rootv = 1.0
data["index_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["index_3_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[7].x * coef
rooty = -lhandlms[7].z * coef
rootz = -lhandlms[7].y * coef
rootv = 1.0
data["index_4_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["index_4_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[10].x * coef
rooty = -lhandlms[10].z * coef
rootz = -lhandlms[10].y * coef
rootv = 1.0
data["middle_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["middle_3_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[11].x * coef
rooty = -lhandlms[11].z * coef
rootz = -lhandlms[11].y * coef
rootv = 1.0
data["middle_4_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["middle_4_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[14].x * coef
rooty = -lhandlms[14].z * coef
rootz = -lhandlms[14].y * coef
rootv = 1.0
data["ring_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["ring_3_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[15].x * coef
rooty = -lhandlms[15].z * coef
rootz = -lhandlms[15].y * coef
rootv = 1.0
data["ring_4_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["ring_4_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[18].x * coef
rooty = -lhandlms[18].z * coef
rootz = -lhandlms[18].y * coef
rootv = 1.0
data["pinky_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["pinky_3_l"] = [rootx, rooty, rootz, rootv]
rootx = lhandlms[19].x * coef
rooty = -lhandlms[19].z * coef
rootz = -lhandlms[19].y * coef
rootv = 1.0
data["pinky_4_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
dataxyz["pinky_4_l"] = [rootx, rooty, rootz, rootv]

81
README.md Normal file
View File

@@ -0,0 +1,81 @@
# Human Pose Estimation with mediapipe
## Запуск
Для вебкамеры, где 0 - индекс самой вебкамеры, если отличается, то следует заменить на нужный.
```console
python3 hpe_videocapture.py 0 scale_pose
```
Для чтения из файла
```console
python3 hpe_videocapture.py path_to_video scale_pose
```
Параметр scale_pose - ширина плеч. По умолчанию равен 0,42.
Есть экспериментальная версия для взаимодействия с ffmpeg и osc:
```console
python3 hpe_ffmpeg.py
```
Внутри кода записаны параметры для osc соединения:
* osc_address = "10.199.0.14"
* osc_port = 5005
Предполагается вывести выше указанные параметры либо в аргументы, либо в конфиг-файл.
## Зависимости
Библиотеки python:
* mediapipe
* numpy
* matplotlib (можно сделать опциональной)
* opencv-python
* json
* python-osc (в файле hpe_ffmpeg.py)
Библиотека mediapipe требует cudatoolkit и cudnn для работы с gpu.
## Интерфейс для hpe_videocapture
Для разовой отрисовки точек в 3D с помощью matplotlib необходимо нажать клавишу 'm'.
Чтобы записать сконвертированные данные в модель UE4 в файл 'hierarchy_data.json', нужно нажать клавишу 'j'.
Для закрытия программы нужно нажать клавишу 'esc'.
## Требования к постановке эксперимента
* Человек в кадре должен быть один.
* Камера примерно должна быть под углом 90 градусов.
* Человек должен входить в кадр целиком.
* Желательно, чтобы у одежда на человеке были хорошо видны контуры.
## Сборка на Windows
В разработке
## Config file
В разработке.
Предположительные переменные.
Параметры класса hpe_mp_class из hpe_mp_class.py:
* hands_static_image_mode = False
* hands_max_num_hands = 2
* hands_min_detection_confidence = 0.7
* hands_min_tracking_confidence = 0.5
* pose_static_image_mode = False
* pose_upper_body_only = False
* pose_smooth_landmarks = True
* pose_min_detection_confidence = 0.7
* pose_min_tracking_confidence = 0.5
* hol_static_image_mode = False
* hol_upper_body_only = False
* hol_smooth_landmarks = True
* hol_min_detection_confidence = 0.7
* hol_min_tracking_confidence = 0.5
Физические параметры человека (ширина плеч в метрах):
* scale_pose = 0,42

66
hpe_ffmpeg.py Normal file
View File

@@ -0,0 +1,66 @@
import sys
import cv2
import time
from hpe_mp_class import hpe_mp_class
from pythonosc import udp_client
# Arguments
adress_input = "in.pipe1"
if len(sys.argv) > 1:
adress_input = sys.argv[1]
for i in range(0, 100):
if adress_input == str(i):
adress_input = i
osc_address = "10.199.0.14"
osc_port = 5005
client = udp_client.SimpleUDPClient(osc_address, osc_port)
showing = False
# Videocapture
cap = cv2.VideoCapture(adress_input)
# Preprocessing parameters
crop = 1.0
frame_width = int(crop*cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(crop*cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
# FPS variables
pTime = 0
cTime = 0
# Mediapie class
mp_cl = hpe_mp_class()
while True:
# Reading frame
success, img = cap.read()
# Image preprocessing
img = cv2.resize(img, (frame_width, frame_height))
# Mediapipe
mp_cl.process(img)
mp_cl.show(img)
if showing:
# FPS
cTime = time.time()
fps = 1. / (cTime - pTime)
pTime = cTime
# Showing
img = cv2.flip(img, 1) # mirror
cv2.putText(img, str(int(fps)), (22, 32), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2)
cv2.putText(img, str(int(fps)), (20, 30), cv2.FONT_HERSHEY_COMPLEX, 1, (255, 255, 255), 2)
cv2.imshow("Main", img)
# get converted map
res = mp_cl.getJSON()
client.send_message("/pose/0", res)
# Interface
key = cv2.waitKey(1)
if key == 27:
break

204
hpe_mp_class.py Normal file
View File

@@ -0,0 +1,204 @@
import cv2
import mediapipe as mp
import numpy as np
mpHands = mp.solutions.hands
mpPose = mp.solutions.pose
mpDraw = mp.solutions.drawing_utils
mpHolistic = mp.solutions.holistic
import logging
FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(filename="hpe_mp_class.log", level=logging.INFO, format=FORMAT)
logger = logging.getLogger("hpe_mp_class_logger")
from ModelUE4 import bodyconvert
from ModelUE4 import rhandconvert
from ModelUE4 import lhandconvert
class hpe_mp_class():
def __init__(self):
hands_static_image_mode = False
hands_max_num_hands = 2
hands_min_detection_confidence = 0.7
hands_min_tracking_confidence = 0.5
pose_static_image_mode = False
pose_upper_body_only = False
pose_smooth_landmarks = True
pose_min_detection_confidence = 0.7
pose_min_tracking_confidence = 0.5
hol_static_image_mode = False
hol_upper_body_only = False
hol_smooth_landmarks = True
hol_min_detection_confidence = 0.7
hol_min_tracking_confidence = 0.5
self.holistic_use = True
self.coef = 1.0
try:
self.hands = mpHands.Hands(hands_static_image_mode,
hands_max_num_hands,
hands_min_detection_confidence,
hands_min_tracking_confidence)
self.poses = mpPose.Pose(pose_static_image_mode,
pose_upper_body_only,
pose_smooth_landmarks,
pose_min_detection_confidence,
pose_min_tracking_confidence)
self.hol = mpHolistic.Holistic(hol_static_image_mode,
hol_upper_body_only,
hol_smooth_landmarks,
hol_min_detection_confidence,
hol_min_tracking_confidence)
logger.info("Success initialize hpe class")
except Exception as err:
logger.exception("Error initialize hpe class: " + str(err))
def process(self, image, scale_pose=0.42):
try:
imgRGB = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
if self.holistic_use:
self.results_hol = self.hol.process(image)
poseslms = {}
if self.results_hol.pose_landmarks:
for id, lm in enumerate(self.results_hol.pose_landmarks.landmark):
poseslms[id] = lm
rast = np.sqrt(pow(poseslms[11].x - poseslms[12].x, 2) + pow(poseslms[11].y - poseslms[12].y, 2) + pow(poseslms[11].z - poseslms[12].z, 2))
self.coef = scale_pose / rast
else:
self.results_hands = self.hands.process(imgRGB)
self.results_pose = self.poses.process(imgRGB)
except Exception as err:
logger.exception("Error processing hpe class: " + str(err))
def show(self, image):
try:
if self.holistic_use:
mpDraw.draw_landmarks(
image,
self.results_hol.face_landmarks,
mpHolistic.FACE_CONNECTIONS)
mpDraw.draw_landmarks(
image,
self.results_hol.pose_landmarks,
mpHolistic.POSE_CONNECTIONS)
mpDraw.draw_landmarks(
image,
self.results_hol.left_hand_landmarks,
mpHands.HAND_CONNECTIONS)
mpDraw.draw_landmarks(
image,
self.results_hol.right_hand_landmarks,
mpHands.HAND_CONNECTIONS)
else:
if self.results_hands.multi_hand_landmarks:
for handLms in self.results_hands.multi_hand_landmarks:
mpDraw.draw_landmarks(image, handLms, mpHands.HAND_CONNECTIONS)
for id, lm in enumerate(handLms.landmark):
h, w, c = image.shape
cx, cy = int(lm.x * w), int(lm.y * h)
if id == 4 or id == 8 or id == 12 or id == 16 or id == 20:
cv2.circle(image, (cx, cy), 10, (255, 0, 0), cv2.FILLED)
if self.results_pose.pose_landmarks:
mpDraw.draw_landmarks(image, self.results_pose.pose_landmarks, mpPose.POSE_CONNECTIONS)
except Exception as err:
logger.exception("Error showing hpe class: " + str(err))
def getResults(self):
res = {}
try:
poseslms = {}
handslms = []
if self.holistic_use:
if self.results_hol.pose_landmarks:
for id, lm in enumerate(self.results_hol.pose_landmarks.landmark):
poseslms[id] = lm
hand = {}
if self.results_hol.left_hand_landmarks:
for id, lm in enumerate(self.results_hol.left_hand_landmarks.landmark):
hand[id] = lm
handslms.append(hand)
if self.results_hol.right_hand_landmarks:
for id, lm in enumerate(self.results_hol.right_hand_landmarks.landmark):
hand[id+21] = lm
handslms.append(hand)
res["poses"] = poseslms
res["hands"] = handslms
else:
if self.results_pose.pose_landmarks:
for id, lm in enumerate(self.results_pose.pose_landmarks.landmark):
poseslms[id] = lm
if self.results_hands.multi_hand_landmarks:
for handLms in self.results_hands.multi_hand_landmarks:
hand = {}
for id, lm in enumerate(handLms.landmark):
hand[id] = lm
handslms.append(hand)
res["poses"] = poseslms
res["hands"] = handslms
except Exception as err:
logger.exception("Error getting result hpe class: " + str(err))
return res
def scaleResult(self, result):
try:
if self.holistic_use:
for pp in result["poses"].keys():
result["poses"][pp].x *= self.coef
result["poses"][pp].y *= self.coef
result["poses"][pp].z *= self.coef
if len(result["hands"]) > 0:
for hp0 in result["hands"][0].keys():
result["hands"][0][hp0].x *= self.coef
result["hands"][0][hp0].y *= self.coef
result["hands"][0][hp0].z *= self.coef
if len(result["hands"]) > 1:
for hp1 in result["hands"][1].keys():
result["hands"][1][hp1].x *= self.coef
result["hands"][1][hp1].y *= self.coef
result["hands"][1][hp1].z *= self.coef
except Exception as err:
logger.exception("Error scaling hpe class: " + str(err))
def getJSON(self):
data = {}
try:
if self.holistic_use:
poseslms = {}
maxy = 0
if self.results_hol.pose_landmarks:
for id, lm in enumerate(self.results_hol.pose_landmarks.landmark):
poseslms[id] = lm
if lm.y > maxy:
maxy = lm.y
bodyconvert(poseslms, data, self.coef, maxy)
rhandlms = {}
if self.results_hol.right_hand_landmarks:
for id, lm in enumerate(self.results_hol.right_hand_landmarks.landmark):
rhandlms[id] = lm
rhandconvert(rhandlms, data, self.coef)
lhandlms = {}
if self.results_hol.left_hand_landmarks:
for id, lm in enumerate(self.results_hol.left_hand_landmarks.landmark):
lhandlms[id] = lm
lhandconvert(lhandlms, data, self.coef)
except Exception as err:
logger.exception("Error json converting hpe class: " + str(err))
return data

90
hpe_videocapture.py Normal file
View File

@@ -0,0 +1,90 @@
import sys
import cv2
import time
from hpe_mp_class import hpe_mp_class
import matplotlib.pyplot as plt
import json
# Arguments
adress_input = sys.argv[1]
for i in range(0, 100):
if adress_input == str(i):
adress_input = i
scale_pose = 0.42
if len(sys.argv) > 2:
scale_pose = float(sys.argv[2])
# Videocapture
cap = cv2.VideoCapture(adress_input)
# Preprocessing parameters
crop = 1.0
frame_width = int(crop*cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(crop*cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
# FPS variables
pTime = 0
cTime = 0
# Mediapie class
mp_cl = hpe_mp_class()
while True:
# Reading frame
success, img = cap.read()
# Image preprocessing
img = cv2.resize(img, (frame_width, frame_height))
# Mediapipe
mp_cl.process(img, scale_pose=scale_pose)
mp_cl.show(img)
# FPS
cTime = time.time()
fps = 1. / (cTime - pTime)
pTime = cTime
# Showing
img = cv2.flip(img, 1) # mirror
cv2.putText(img, str(int(fps)), (22, 32), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2)
cv2.putText(img, str(int(fps)), (20, 30), cv2.FONT_HERSHEY_COMPLEX, 1, (255, 255, 255), 2)
cv2.imshow("Main", img)
# Interface
key = cv2.waitKey(1)
if key == 27:
break
if key == 109:
# Matplotlib
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')
res = mp_cl.getResults()
mp_cl.scaleResult(res)
points_body_x = []
points_body_y = []
points_body_z = []
points_hand_x = []
points_hand_y = []
points_hand_z = []
for pp in res["poses"].values():
if pp.visibility > 0.9:
points_body_x.append(pp.x)
points_body_y.append(pp.y)
points_body_z.append(pp.z)
for hand in res["hands"]:
for hp in hand.values():
points_hand_x.append(hp.x)
points_hand_y.append(hp.y)
points_hand_z.append(hp.z)
ax.scatter(points_body_x, points_body_y, points_body_z, color='blue')
ax.scatter(points_hand_x, points_hand_y, points_hand_z, color='green')
plt.show()
if key == 106:
# JSON
res = mp_cl.getJSON()
with open('hierarchy_data.json', 'w', encoding='utf-8') as f:
json.dump(res, f, ensure_ascii=False, indent=4)