First commit

.gitignore

@@ -0,0 +1,134 @@
+# ---> Python
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+datacndm
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+.idea*
+*.log

ModelUE4.py

@@ -0,0 +1,479 @@
+# convert mediapipe model to model from Unreal
+
+# element of json
+def makexyzrotvis(x,y,z,rotx,roty,rotz,visible):
+    ret = {}
+    translation = {}
+    rotation = {}
+    scale = {}
+    translation["x"] = x
+    translation["y"] = y
+    translation["z"] = z
+    rotation["rotx"] = rotx
+    rotation["roty"] = roty
+    rotation["rotz"] = rotz
+    scale["x"] = 1.0
+    scale["y"] = 1.0
+    scale["z"] = 1.0
+    ret["visible"] = visible
+    ret["translation"] = translation
+    ret["rotation"] = rotation
+    ret["scale"] = scale
+    return ret
+
+def bodyconvert(poseslms, data, coef, maxy):
+    dataxyz = {}
+
+    rootx = poseslms[24].x * coef
+    rooty = -poseslms[24].z * coef
+    rootz = -poseslms[24].y * coef
+    rootv = poseslms[24].visibility
+    data["hip_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["hip_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[23].x * coef
+    rooty = -poseslms[23].z * coef
+    rootz = -poseslms[23].y * coef
+    rootv = poseslms[23].visibility
+    data["hip_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["hip_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = 0.5 * (dataxyz["hip_l"][0] + dataxyz["hip_r"][0]) * coef
+    rooty = -maxy * coef
+    rootz = -0.5 * (dataxyz["hip_l"][2] + dataxyz["hip_r"][2]) * coef
+    rootv = 1.0
+    data["root"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["root"] = [rootx, rooty, rootz, rootv]
+
+    rootx = 0.5 * (dataxyz["hip_l"][0] + dataxyz["hip_r"][0]) * coef
+    rooty = -0.5 * (dataxyz["hip_l"][1] + dataxyz["hip_r"][1]) * coef
+    rootz = -0.5 * (dataxyz["hip_l"][2] + dataxyz["hip_r"][2]) * coef
+    rootv = 0.5 * (dataxyz["hip_l"][3] + dataxyz["hip_r"][3])
+    data["pelvis"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["pelvis"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[0].x * coef
+    rooty = -poseslms[0].z * coef
+    rootz = -poseslms[0].y * coef
+    rootv = poseslms[0].visibility
+    data["head"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["head"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.955887 * dataxyz["head"][0] + 0.044113 * dataxyz["pelvis"][0]) * coef
+    rooty = (0.955887 * dataxyz["head"][1] + 0.044113 * dataxyz["pelvis"][1]) * coef
+    rootz = (0.955887 * dataxyz["head"][2] + 0.044113 * dataxyz["pelvis"][2]) * coef
+    rootv = (0.955887 * dataxyz["head"][3] + 0.044113 * dataxyz["pelvis"][3])
+    data["spine_01"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["spine_01"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.864201 * dataxyz["head"][0] + 0.135799 * dataxyz["pelvis"][0]) * coef
+    rooty = (0.864201 * dataxyz["head"][1] + 0.135799 * dataxyz["pelvis"][1]) * coef
+    rootz = (0.864201 * dataxyz["head"][2] + 0.135799 * dataxyz["pelvis"][2]) * coef
+    rootv = (0.864201 * dataxyz["head"][3] + 0.135799 * dataxyz["pelvis"][3])
+    data["spine_02"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["spine_02"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.717787 * dataxyz["head"][0] + 0.282213 * dataxyz["pelvis"][0]) * coef
+    rooty = (0.717787 * dataxyz["head"][1] + 0.282213 * dataxyz["pelvis"][1]) * coef
+    rootz = (0.717787 * dataxyz["head"][2] + 0.282213 * dataxyz["pelvis"][2]) * coef
+    rootv = (0.717787 * dataxyz["head"][3] + 0.282213 * dataxyz["pelvis"][3])
+    data["spine_03"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["spine_03"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.548406 * dataxyz["head"][0] + 0.451594 * dataxyz["pelvis"][0]) * coef
+    rooty = (0.548406 * dataxyz["head"][1] + 0.451594 * dataxyz["pelvis"][1]) * coef
+    rootz = (0.548406 * dataxyz["head"][2] + 0.451594 * dataxyz["pelvis"][2]) * coef
+    rootv = (0.548406 * dataxyz["head"][3] + 0.451594 * dataxyz["pelvis"][3])
+    data["spine_04"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["spine_04"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.228975 * dataxyz["head"][0] + 0.771025 * dataxyz["pelvis"][0]) * coef
+    rooty = (0.228975 * dataxyz["head"][1] + 0.771025 * dataxyz["pelvis"][1]) * coef
+    rootz = (0.228975 * dataxyz["head"][2] + 0.771025 * dataxyz["pelvis"][2]) * coef
+    rootv = (0.228975 * dataxyz["head"][3] + 0.771025 * dataxyz["pelvis"][3])
+    data["spine_05"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["spine_05"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[26].x * coef
+    rooty = -poseslms[26].z * coef
+    rootz = -poseslms[26].y * coef
+    rootv = poseslms[26].visibility
+    data["knee_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["knee_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[25].x * coef
+    rooty = -poseslms[25].z * coef
+    rootz = -poseslms[25].y * coef
+    rootv = poseslms[25].visibility
+    data["knee_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["knee_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[28].x * coef
+    rooty = -poseslms[28].z * coef
+    rootz = -poseslms[28].y * coef
+    rootv = poseslms[28].visibility
+    data["ankle_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["ankle_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[27].x * coef
+    rooty = -poseslms[27].z * coef
+    rootz = -poseslms[27].y * coef
+    rootv = poseslms[27].visibility
+    data["ankle_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["ankle_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[32].x * coef
+    rooty = -poseslms[32].z * coef
+    rootz = -poseslms[32].y * coef
+    rootv = poseslms[32].visibility
+    data["ball_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["ball_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[31].x * coef
+    rooty = -poseslms[31].z * coef
+    rootz = -poseslms[31].y * coef
+    rootv = poseslms[31].visibility
+    data["ball_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["ball_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[30].x * coef
+    rooty = -poseslms[30].z * coef
+    rootz = -poseslms[30].y * coef
+    rootv = poseslms[30].visibility
+    data["heel_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["heel_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[29].x * coef
+    rooty = -poseslms[29].z * coef
+    rootz = -poseslms[29].y * coef
+    rootv = poseslms[29].visibility
+    data["heel_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["heel_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[12].x * coef
+    rooty = -poseslms[12].z * coef
+    rootz = -poseslms[12].y * coef
+    rootv = poseslms[12].visibility
+    data["shoulder_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["shoulder_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[11].x * coef
+    rooty = -poseslms[11].z * coef
+    rootz = -poseslms[11].y * coef
+    rootv = poseslms[11].visibility
+    data["shoulder_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["shoulder_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.635 * dataxyz["shoulder_r"][0] + 0.365 * dataxyz["shoulder_l"][0]) * coef
+    rooty = 0.5 * (0.635 * dataxyz["shoulder_r"][1] + 0.365 * dataxyz["shoulder_l"][1] + 0.31 * dataxyz["head"][1] + 0.69 * dataxyz["spine_05"][1]) * coef
+    rootz = (0.0635 * dataxyz["shoulder_r"][1] + 0.0365 * dataxyz["shoulder_l"][1] + 0.9 * dataxyz["spine_05"][1]) * coef
+    rootv = (0.635 * dataxyz["shoulder_r"][0] + 0.451594 * dataxyz["shoulder_l"][0])
+    data["clavicle_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["clavicle_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.635 * dataxyz["shoulder_l"][0] + 0.365 * dataxyz["shoulder_r"][0]) * coef
+    rooty = 0.5 * (0.635 * dataxyz["shoulder_l"][1] + 0.365 * dataxyz["shoulder_r"][1] + 0.31 * dataxyz["head"][1] + 0.69 * dataxyz["spine_05"][1]) * coef
+    rootz = (0.0635 * dataxyz["shoulder_l"][1] + 0.0365 * dataxyz["shoulder_r"][1] + 0.9 * dataxyz["spine_05"][1]) * coef
+    rootv = (0.635 * dataxyz["shoulder_l"][0] + 0.365 * dataxyz["shoulder_r"][0])
+    data["clavicle_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["clavicle_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[14].x * coef
+    rooty = -poseslms[14].z * coef
+    rootz = -poseslms[14].y * coef
+    rootv = poseslms[14].visibility
+    data["elbow_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["elbow_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = poseslms[13].x * coef
+    rooty = -poseslms[13].z * coef
+    rootz = -poseslms[13].y * coef
+    rootv = poseslms[13].visibility
+    data["elbow_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["elbow_l"] = [rootx, rooty, rootz, rootv]
+
+def rhandconvert(rhandlms, data, coef):
+    dataxyz = {}
+
+    rootx = rhandlms[0].x * coef
+    rooty = -rhandlms[0].z * coef
+    rootz = -rhandlms[0].y * coef
+    rootv = 1.0
+    data["hand_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["hand_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[5].x * coef
+    rooty = -rhandlms[5].z * coef
+    rootz = -rhandlms[5].y * coef
+    rootv = 1.0
+    data["index_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["index_2_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.75 * dataxyz["hand_r"][0] + 0.25 * dataxyz["index_2_r"][0]) * coef
+    rooty = (0.75 * dataxyz["hand_r"][1] + 0.25 * dataxyz["index_2_r"][1]) * coef
+    rootz = (0.75 * dataxyz["hand_r"][2] + 0.25 * dataxyz["index_2_r"][2]) * coef
+    rootv = 1.0
+    data["index_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["index_1_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[9].x * coef
+    rooty = -rhandlms[9].z * coef
+    rootz = -rhandlms[9].y * coef
+    rootv = 1.0
+    data["middle_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["middle_2_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.75 * dataxyz["hand_r"][0] + 0.25 * dataxyz["middle_2_r"][0]) * coef
+    rooty = (0.75 * dataxyz["hand_r"][1] + 0.25 * dataxyz["middle_2_r"][1]) * coef
+    rootz = (0.75 * dataxyz["hand_r"][2] + 0.25 * dataxyz["middle_2_r"][2]) * coef
+    rootv = 1.0
+    data["middle_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["middle_1_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[13].x * coef
+    rooty = -rhandlms[13].z * coef
+    rootz = -rhandlms[13].y * coef
+    rootv = 1.0
+    data["ring_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["ring_2_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.75 * dataxyz["hand_r"][0] + 0.25 * dataxyz["ring_2_r"][0]) * coef
+    rooty = (0.75 * dataxyz["hand_r"][1] + 0.25 * dataxyz["ring_2_r"][1]) * coef
+    rootz = (0.75 * dataxyz["hand_r"][2] + 0.25 * dataxyz["ring_2_r"][2]) * coef
+    rootv = 1.0
+    data["ring_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["ring_1_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[17].x * coef
+    rooty = -rhandlms[17].z * coef
+    rootz = -rhandlms[17].y * coef
+    rootv = 1.0
+    data["pinky_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["pinky_2_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.75 * dataxyz["hand_r"][0] + 0.25 * dataxyz["pinky_2_r"][0]) * coef
+    rooty = (0.75 * dataxyz["hand_r"][1] + 0.25 * dataxyz["pinky_2_r"][1]) * coef
+    rootz = (0.75 * dataxyz["hand_r"][2] + 0.25 * dataxyz["pinky_2_r"][2]) * coef
+    rootv = 1.0
+    data["pinky_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["pinky_1_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[1].x * coef
+    rooty = -rhandlms[1].z * coef
+    rootz = -rhandlms[1].y * coef
+    rootv = 1.0
+    data["thumb_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["thumb_1_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[2].x * coef
+    rooty = -rhandlms[2].z * coef
+    rootz = -rhandlms[2].y * coef
+    rootv = 1.0
+    data["thumb_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["thumb_2_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[3].x * coef
+    rooty = -rhandlms[3].z * coef
+    rootz = -rhandlms[3].y * coef
+    rootv = 1.0
+    data["thumb_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["thumb_3_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[6].x * coef
+    rooty = -rhandlms[6].z * coef
+    rootz = -rhandlms[6].y * coef
+    rootv = 1.0
+    data["index_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["index_3_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[7].x * coef
+    rooty = -rhandlms[7].z * coef
+    rootz = -rhandlms[7].y * coef
+    rootv = 1.0
+    data["index_4_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["index_4_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[10].x * coef
+    rooty = -rhandlms[10].z * coef
+    rootz = -rhandlms[10].y * coef
+    rootv = 1.0
+    data["middle_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["middle_3_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[11].x * coef
+    rooty = -rhandlms[11].z * coef
+    rootz = -rhandlms[11].y * coef
+    rootv = 1.0
+    data["middle_4_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["middle_4_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[14].x * coef
+    rooty = -rhandlms[14].z * coef
+    rootz = -rhandlms[14].y * coef
+    rootv = 1.0
+    data["ring_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["ring_3_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[15].x * coef
+    rooty = -rhandlms[15].z * coef
+    rootz = -rhandlms[15].y * coef
+    rootv = 1.0
+    data["ring_4_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["ring_4_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[18].x * coef
+    rooty = -rhandlms[18].z * coef
+    rootz = -rhandlms[18].y * coef
+    rootv = 1.0
+    data["pinky_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["pinky_3_r"] = [rootx, rooty, rootz, rootv]
+
+    rootx = rhandlms[19].x * coef
+    rooty = -rhandlms[19].z * coef
+    rootz = -rhandlms[19].y * coef
+    rootv = 1.0
+    data["pinky_4_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["pinky_4_r"] = [rootx, rooty, rootz, rootv]
+
+def lhandconvert(lhandlms, data, coef):
+    dataxyz = {}
+
+    rootx = lhandlms[0].x * coef
+    rooty = -lhandlms[0].z * coef
+    rootz = -lhandlms[0].y * coef
+    rootv = 1.0
+    data["hand_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["hand_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[5].x * coef
+    rooty = -lhandlms[5].z * coef
+    rootz = -lhandlms[5].y * coef
+    rootv = 1.0
+    data["index_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["index_2_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.75 * dataxyz["hand_l"][0] + 0.25 * dataxyz["index_2_l"][0]) * coef
+    rooty = (0.75 * dataxyz["hand_l"][1] + 0.25 * dataxyz["index_2_l"][1]) * coef
+    rootz = (0.75 * dataxyz["hand_l"][2] + 0.25 * dataxyz["index_2_l"][2]) * coef
+    rootv = 1.0
+    data["index_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["index_1_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[9].x * coef
+    rooty = -lhandlms[9].z * coef
+    rootz = -lhandlms[9].y * coef
+    rootv = 1.0
+    data["middle_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["middle_2_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.75 * dataxyz["hand_l"][0] + 0.25 * dataxyz["middle_2_l"][0]) * coef
+    rooty = (0.75 * dataxyz["hand_l"][1] + 0.25 * dataxyz["middle_2_l"][1]) * coef
+    rootz = (0.75 * dataxyz["hand_l"][2] + 0.25 * dataxyz["middle_2_l"][2]) * coef
+    rootv = 1.0
+    data["middle_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["middle_1_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[13].x * coef
+    rooty = -lhandlms[13].z * coef
+    rootz = -lhandlms[13].y * coef
+    rootv = 1.0
+    data["ring_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["ring_2_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.75 * dataxyz["hand_l"][0] + 0.25 * dataxyz["ring_2_l"][0]) * coef
+    rooty = (0.75 * dataxyz["hand_l"][1] + 0.25 * dataxyz["ring_2_l"][1]) * coef
+    rootz = (0.75 * dataxyz["hand_l"][2] + 0.25 * dataxyz["ring_2_l"][2]) * coef
+    rootv = 1.0
+    data["ring_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["ring_1_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[17].x * coef
+    rooty = -lhandlms[17].z * coef
+    rootz = -lhandlms[17].y * coef
+    rootv = 1.0
+    data["pinky_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["pinky_2_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = (0.75 * dataxyz["hand_l"][0] + 0.25 * dataxyz["pinky_2_l"][0]) * coef
+    rooty = (0.75 * dataxyz["hand_l"][1] + 0.25 * dataxyz["pinky_2_l"][1]) * coef
+    rootz = (0.75 * dataxyz["hand_l"][2] + 0.25 * dataxyz["pinky_2_l"][2]) * coef
+    rootv = 1.0
+    data["pinky_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["pinky_1_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[1].x * coef
+    rooty = -lhandlms[1].z * coef
+    rootz = -lhandlms[1].y * coef
+    rootv = 1.0
+    data["thumb_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["thumb_1_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[2].x * coef
+    rooty = -lhandlms[2].z * coef
+    rootz = -lhandlms[2].y * coef
+    rootv = 1.0
+    data["thumb_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["thumb_2_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[3].x * coef
+    rooty = -lhandlms[3].z * coef
+    rootz = -lhandlms[3].y * coef
+    rootv = 1.0
+    data["thumb_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["thumb_3_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[6].x * coef
+    rooty = -lhandlms[6].z * coef
+    rootz = -lhandlms[6].y * coef
+    rootv = 1.0
+    data["index_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["index_3_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[7].x * coef
+    rooty = -lhandlms[7].z * coef
+    rootz = -lhandlms[7].y * coef
+    rootv = 1.0
+    data["index_4_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["index_4_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[10].x * coef
+    rooty = -lhandlms[10].z * coef
+    rootz = -lhandlms[10].y * coef
+    rootv = 1.0
+    data["middle_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["middle_3_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[11].x * coef
+    rooty = -lhandlms[11].z * coef
+    rootz = -lhandlms[11].y * coef
+    rootv = 1.0
+    data["middle_4_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["middle_4_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[14].x * coef
+    rooty = -lhandlms[14].z * coef
+    rootz = -lhandlms[14].y * coef
+    rootv = 1.0
+    data["ring_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["ring_3_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[15].x * coef
+    rooty = -lhandlms[15].z * coef
+    rootz = -lhandlms[15].y * coef
+    rootv = 1.0
+    data["ring_4_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["ring_4_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[18].x * coef
+    rooty = -lhandlms[18].z * coef
+    rootz = -lhandlms[18].y * coef
+    rootv = 1.0
+    data["pinky_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["pinky_3_l"] = [rootx, rooty, rootz, rootv]
+
+    rootx = lhandlms[19].x * coef
+    rooty = -lhandlms[19].z * coef
+    rootz = -lhandlms[19].y * coef
+    rootv = 1.0
+    data["pinky_4_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv)
+    dataxyz["pinky_4_l"] = [rootx, rooty, rootz, rootv]

README.md

@@ -0,0 +1,81 @@
+# Human Pose Estimation with mediapipe
+
+## Запуск
+
+Для вебкамеры, где 0 - индекс самой вебкамеры, если отличается, то следует заменить на нужный.
+```console
+python3 hpe_videocapture.py 0 scale_pose
+```
+
+Для чтения из файла
+```console
+python3 hpe_videocapture.py path_to_video scale_pose
+```
+
+Параметр scale_pose - ширина плеч. По умолчанию равен 0,42.
+
+Есть экспериментальная версия для взаимодействия с ffmpeg и osc:
+```console
+python3 hpe_ffmpeg.py 
+```
+Внутри кода записаны параметры для osc соединения:
+* osc_address = "10.199.0.14"
+* osc_port = 5005
+
+Предполагается вывести выше указанные параметры либо в аргументы, либо в конфиг-файл.
+
+## Зависимости
+
+Библиотеки python:
+* mediapipe
+* numpy
+* matplotlib (можно сделать опциональной)
+* opencv-python
+* json
+* python-osc (в файле hpe_ffmpeg.py)
+
+Библиотека mediapipe требует cudatoolkit и cudnn для работы с gpu.
+
+## Интерфейс для hpe_videocapture
+
+Для разовой отрисовки точек в 3D с помощью matplotlib необходимо нажать клавишу 'm'.
+
+Чтобы записать сконвертированные данные в модель UE4 в файл 'hierarchy_data.json', нужно нажать клавишу 'j'.
+
+Для закрытия программы нужно нажать клавишу 'esc'.
+
+## Требования к постановке эксперимента
+
+* Человек в кадре должен быть один.
+* Камера примерно должна быть под углом 90 градусов.
+* Человек должен входить в кадр целиком.
+* Желательно, чтобы у одежда на человеке были хорошо видны контуры. 
+
+## Сборка на Windows
+
+В разработке
+
+## Config file
+
+В разработке.
+
+Предположительные переменные.
+
+Параметры класса hpe_mp_class из hpe_mp_class.py:
+* hands_static_image_mode = False
+* hands_max_num_hands = 2
+* hands_min_detection_confidence = 0.7
+* hands_min_tracking_confidence = 0.5
+* pose_static_image_mode = False
+* pose_upper_body_only = False
+* pose_smooth_landmarks = True
+* pose_min_detection_confidence = 0.7
+* pose_min_tracking_confidence = 0.5
+* hol_static_image_mode = False
+* hol_upper_body_only = False
+* hol_smooth_landmarks = True
+* hol_min_detection_confidence = 0.7
+* hol_min_tracking_confidence = 0.5
+
+Физические параметры человека (ширина плеч в метрах):
+* scale_pose = 0,42

hpe_ffmpeg.py

@@ -0,0 +1,66 @@
+import sys
+import cv2
+import time
+from hpe_mp_class import hpe_mp_class
+from pythonosc import udp_client
+
+# Arguments
+adress_input = "in.pipe1"
+if len(sys.argv) > 1:
+    adress_input = sys.argv[1]
+    for i in range(0, 100):
+        if adress_input == str(i):
+            adress_input = i
+
+osc_address = "10.199.0.14"
+osc_port = 5005
+client = udp_client.SimpleUDPClient(osc_address, osc_port)
+
+showing = False
+
+# Videocapture
+cap = cv2.VideoCapture(adress_input)
+
+# Preprocessing parameters
+crop = 1.0
+frame_width = int(crop*cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+frame_height = int(crop*cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+# FPS variables
+pTime = 0
+cTime = 0
+
+# Mediapie class
+mp_cl = hpe_mp_class()
+
+while True:
+    # Reading frame
+    success, img = cap.read()
+
+    # Image preprocessing
+    img = cv2.resize(img, (frame_width, frame_height))
+
+    # Mediapipe
+    mp_cl.process(img)
+    mp_cl.show(img)
+
+    if showing:
+        # FPS
+        cTime = time.time()
+        fps = 1. / (cTime - pTime)
+        pTime = cTime
+
+        # Showing
+        img = cv2.flip(img, 1)  # mirror
+        cv2.putText(img, str(int(fps)), (22, 32), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2)
+        cv2.putText(img, str(int(fps)), (20, 30), cv2.FONT_HERSHEY_COMPLEX, 1, (255, 255, 255), 2)
+        cv2.imshow("Main", img)
+
+    # get converted map
+    res = mp_cl.getJSON()
+    client.send_message("/pose/0", res)
+
+    # Interface
+    key = cv2.waitKey(1)
+    if key == 27:
+        break

hpe_mp_class.py

@@ -0,0 +1,204 @@
+import cv2
+import mediapipe as mp
+import numpy as np
+
+mpHands = mp.solutions.hands
+mpPose = mp.solutions.pose
+mpDraw = mp.solutions.drawing_utils
+mpHolistic = mp.solutions.holistic
+
+import logging
+FORMAT = '%(asctime)-15s %(message)s'
+logging.basicConfig(filename="hpe_mp_class.log", level=logging.INFO, format=FORMAT)
+logger = logging.getLogger("hpe_mp_class_logger")
+
+from ModelUE4 import bodyconvert
+from ModelUE4 import rhandconvert
+from ModelUE4 import lhandconvert
+
+class hpe_mp_class():
+
+    def __init__(self):
+        hands_static_image_mode = False
+        hands_max_num_hands = 2
+        hands_min_detection_confidence = 0.7
+        hands_min_tracking_confidence = 0.5
+
+        pose_static_image_mode = False
+        pose_upper_body_only = False
+        pose_smooth_landmarks = True
+        pose_min_detection_confidence = 0.7
+        pose_min_tracking_confidence = 0.5
+
+        hol_static_image_mode = False
+        hol_upper_body_only = False
+        hol_smooth_landmarks = True
+        hol_min_detection_confidence = 0.7
+        hol_min_tracking_confidence = 0.5
+
+        self.holistic_use = True
+        self.coef = 1.0
+
+        try:
+            self.hands = mpHands.Hands(hands_static_image_mode,
+                                       hands_max_num_hands,
+                                       hands_min_detection_confidence,
+                                       hands_min_tracking_confidence)
+            self.poses = mpPose.Pose(pose_static_image_mode,
+                                     pose_upper_body_only,
+                                     pose_smooth_landmarks,
+                                     pose_min_detection_confidence,
+                                     pose_min_tracking_confidence)
+            self.hol = mpHolistic.Holistic(hol_static_image_mode,
+                                           hol_upper_body_only,
+                                           hol_smooth_landmarks,
+                                           hol_min_detection_confidence,
+                                           hol_min_tracking_confidence)
+            logger.info("Success initialize hpe class")
+        except Exception as err:
+            logger.exception("Error initialize hpe class: " + str(err))
+
+    def process(self, image, scale_pose=0.42):
+        try:
+            imgRGB = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+
+            if self.holistic_use:
+                self.results_hol = self.hol.process(image)
+
+                poseslms = {}
+                if self.results_hol.pose_landmarks:
+                    for id, lm in enumerate(self.results_hol.pose_landmarks.landmark):
+                        poseslms[id] = lm
+                    rast = np.sqrt(pow(poseslms[11].x - poseslms[12].x, 2) + pow(poseslms[11].y - poseslms[12].y, 2) + pow(poseslms[11].z - poseslms[12].z, 2))
+                    self.coef = scale_pose / rast
+            else:
+                self.results_hands = self.hands.process(imgRGB)
+                self.results_pose = self.poses.process(imgRGB)
+        except Exception as err:
+            logger.exception("Error processing hpe class: " + str(err))
+
+    def show(self, image):
+        try:
+            if self.holistic_use:
+                mpDraw.draw_landmarks(
+                    image,
+                    self.results_hol.face_landmarks,
+                    mpHolistic.FACE_CONNECTIONS)
+                mpDraw.draw_landmarks(
+                    image,
+                    self.results_hol.pose_landmarks,
+                    mpHolistic.POSE_CONNECTIONS)
+                mpDraw.draw_landmarks(
+                    image,
+                    self.results_hol.left_hand_landmarks,
+                    mpHands.HAND_CONNECTIONS)
+                mpDraw.draw_landmarks(
+                    image,
+                    self.results_hol.right_hand_landmarks,
+                    mpHands.HAND_CONNECTIONS)
+            else:
+                if self.results_hands.multi_hand_landmarks:
+                    for handLms in self.results_hands.multi_hand_landmarks:
+                        mpDraw.draw_landmarks(image, handLms, mpHands.HAND_CONNECTIONS)
+                        for id, lm in enumerate(handLms.landmark):
+                            h, w, c = image.shape
+                            cx, cy = int(lm.x * w), int(lm.y * h)
+                            if id == 4 or id == 8 or id == 12 or id == 16 or id == 20:
+                                cv2.circle(image, (cx, cy), 10, (255, 0, 0), cv2.FILLED)
+
+                if self.results_pose.pose_landmarks:
+                    mpDraw.draw_landmarks(image, self.results_pose.pose_landmarks, mpPose.POSE_CONNECTIONS)
+        except Exception as err:
+            logger.exception("Error showing hpe class: " + str(err))
+
+    def getResults(self):
+        res = {}
+        try:
+            poseslms = {}
+            handslms = []
+
+            if self.holistic_use:
+                if self.results_hol.pose_landmarks:
+                    for id, lm in enumerate(self.results_hol.pose_landmarks.landmark):
+                        poseslms[id] = lm
+                hand = {}
+                if self.results_hol.left_hand_landmarks:
+                    for id, lm in enumerate(self.results_hol.left_hand_landmarks.landmark):
+                        hand[id] = lm
+                    handslms.append(hand)
+                if self.results_hol.right_hand_landmarks:
+                    for id, lm in enumerate(self.results_hol.right_hand_landmarks.landmark):
+                        hand[id+21] = lm
+                    handslms.append(hand)
+                res["poses"] = poseslms
+                res["hands"] = handslms
+            else:
+                if self.results_pose.pose_landmarks:
+                    for id, lm in enumerate(self.results_pose.pose_landmarks.landmark):
+                        poseslms[id] = lm
+                if self.results_hands.multi_hand_landmarks:
+                    for handLms in self.results_hands.multi_hand_landmarks:
+                        hand = {}
+                        for id, lm in enumerate(handLms.landmark):
+                            hand[id] = lm
+                        handslms.append(hand)
+                res["poses"] = poseslms
+                res["hands"] = handslms
+        except Exception as err:
+            logger.exception("Error getting result hpe class: " + str(err))
+
+        return res
+
+    def scaleResult(self, result):
+        try:
+            if self.holistic_use:
+                for pp in result["poses"].keys():
+                    result["poses"][pp].x *= self.coef
+                    result["poses"][pp].y *= self.coef
+                    result["poses"][pp].z *= self.coef
+
+                if len(result["hands"]) > 0:
+                    for hp0 in result["hands"][0].keys():
+                        result["hands"][0][hp0].x *= self.coef
+                        result["hands"][0][hp0].y *= self.coef
+                        result["hands"][0][hp0].z *= self.coef
+
+                if len(result["hands"]) > 1:
+                    for hp1 in result["hands"][1].keys():
+                        result["hands"][1][hp1].x *= self.coef
+                        result["hands"][1][hp1].y *= self.coef
+                        result["hands"][1][hp1].z *= self.coef
+        except Exception as err:
+            logger.exception("Error scaling hpe class: " + str(err))
+
+    def getJSON(self):
+        data = {}
+        try:
+            if self.holistic_use:
+                poseslms = {}
+                maxy = 0
+                if self.results_hol.pose_landmarks:
+                    for id, lm in enumerate(self.results_hol.pose_landmarks.landmark):
+                        poseslms[id] = lm
+                        if lm.y > maxy:
+                            maxy = lm.y
+
+                    bodyconvert(poseslms, data, self.coef, maxy)
+
+                rhandlms = {}
+                if self.results_hol.right_hand_landmarks:
+                    for id, lm in enumerate(self.results_hol.right_hand_landmarks.landmark):
+                        rhandlms[id] = lm
+
+                    rhandconvert(rhandlms, data, self.coef)
+
+                lhandlms = {}
+                if self.results_hol.left_hand_landmarks:
+                    for id, lm in enumerate(self.results_hol.left_hand_landmarks.landmark):
+                        lhandlms[id] = lm
+
+                    lhandconvert(lhandlms, data, self.coef)
+        except Exception as err:
+            logger.exception("Error json converting hpe class: " + str(err))
+
+        return data

hpe_videocapture.py

@@ -0,0 +1,90 @@
+import sys
+import cv2
+import time
+from hpe_mp_class import hpe_mp_class
+import matplotlib.pyplot as plt
+import json
+
+# Arguments
+adress_input = sys.argv[1]
+for i in range(0, 100):
+    if adress_input == str(i):
+        adress_input = i
+scale_pose = 0.42
+if len(sys.argv) > 2:
+    scale_pose = float(sys.argv[2])
+
+# Videocapture
+cap = cv2.VideoCapture(adress_input)
+
+# Preprocessing parameters
+crop = 1.0
+frame_width = int(crop*cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+frame_height = int(crop*cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+# FPS variables
+pTime = 0
+cTime = 0
+
+# Mediapie class
+mp_cl = hpe_mp_class()
+
+while True:
+    # Reading frame
+    success, img = cap.read()
+
+    # Image preprocessing
+    img = cv2.resize(img, (frame_width, frame_height))
+
+    # Mediapipe
+    mp_cl.process(img, scale_pose=scale_pose)
+    mp_cl.show(img)
+
+    # FPS
+    cTime = time.time()
+    fps = 1. / (cTime - pTime)
+    pTime = cTime
+
+    # Showing
+    img = cv2.flip(img, 1)  # mirror
+    cv2.putText(img, str(int(fps)), (22, 32), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2)
+    cv2.putText(img, str(int(fps)), (20, 30), cv2.FONT_HERSHEY_COMPLEX, 1, (255, 255, 255), 2)
+    cv2.imshow("Main", img)
+
+    # Interface
+    key = cv2.waitKey(1)
+    if key == 27:
+        break
+    if key == 109:
+        # Matplotlib
+        fig = plt.figure()
+        ax = fig.add_subplot(111, projection='3d')
+        ax.set_xlabel('X')
+        ax.set_ylabel('Y')
+        ax.set_zlabel('Z')
+        res = mp_cl.getResults()
+        mp_cl.scaleResult(res)
+        points_body_x = []
+        points_body_y = []
+        points_body_z = []
+        points_hand_x = []
+        points_hand_y = []
+        points_hand_z = []
+        for pp in res["poses"].values():
+            if pp.visibility > 0.9:
+                points_body_x.append(pp.x)
+                points_body_y.append(pp.y)
+                points_body_z.append(pp.z)
+        for hand in res["hands"]:
+            for hp in hand.values():
+                points_hand_x.append(hp.x)
+                points_hand_y.append(hp.y)
+                points_hand_z.append(hp.z)
+        ax.scatter(points_body_x, points_body_y, points_body_z, color='blue')
+        ax.scatter(points_hand_x, points_hand_y, points_hand_z, color='green')
+        plt.show()
+    if key == 106:
+        # JSON
+        res = mp_cl.getJSON()
+        with open('hierarchy_data.json', 'w', encoding='utf-8') as f:
+            json.dump(res, f, ensure_ascii=False, indent=4)