commit 5459e0f0ce44b6ca56aafb8d67f3254d222aa9ce Author: Arkadiy Strelnikov Date: Fri Nov 26 01:47:03 2021 +0700 First commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b79889e --- /dev/null +++ b/.gitignore @@ -0,0 +1,134 @@ +# ---> Python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +datacndm +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +.idea* +*.log diff --git a/ModelUE4.py b/ModelUE4.py new file mode 100644 index 0000000..b4af9bc --- /dev/null +++ b/ModelUE4.py @@ -0,0 +1,479 @@ +# convert mediapipe model to model from Unreal + +# element of json +def makexyzrotvis(x,y,z,rotx,roty,rotz,visible): + ret = {} + translation = {} + rotation = {} + scale = {} + translation["x"] = x + translation["y"] = y + translation["z"] = z + rotation["rotx"] = rotx + rotation["roty"] = roty + rotation["rotz"] = rotz + scale["x"] = 1.0 + scale["y"] = 1.0 + scale["z"] = 1.0 + ret["visible"] = visible + ret["translation"] = translation + ret["rotation"] = rotation + ret["scale"] = scale + return ret + +def bodyconvert(poseslms, data, coef, maxy): + dataxyz = {} + + rootx = poseslms[24].x * coef + rooty = -poseslms[24].z * coef + rootz = -poseslms[24].y * coef + rootv = poseslms[24].visibility + data["hip_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["hip_r"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[23].x * coef + rooty = -poseslms[23].z * coef + rootz = -poseslms[23].y * coef + rootv = poseslms[23].visibility + data["hip_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["hip_l"] = [rootx, rooty, rootz, rootv] + + rootx = 0.5 * (dataxyz["hip_l"][0] + dataxyz["hip_r"][0]) * coef + rooty = -maxy * coef + rootz = -0.5 * (dataxyz["hip_l"][2] + dataxyz["hip_r"][2]) * coef + rootv = 1.0 + data["root"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["root"] = [rootx, rooty, rootz, rootv] + + rootx = 0.5 * (dataxyz["hip_l"][0] + dataxyz["hip_r"][0]) * coef + rooty = -0.5 * (dataxyz["hip_l"][1] + dataxyz["hip_r"][1]) * coef + rootz = -0.5 * (dataxyz["hip_l"][2] + dataxyz["hip_r"][2]) * coef + rootv = 0.5 * (dataxyz["hip_l"][3] + dataxyz["hip_r"][3]) + data["pelvis"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["pelvis"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[0].x * coef + rooty = -poseslms[0].z * coef + rootz = -poseslms[0].y * coef + rootv = poseslms[0].visibility + data["head"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["head"] = [rootx, rooty, rootz, rootv] + + rootx = (0.955887 * dataxyz["head"][0] + 0.044113 * dataxyz["pelvis"][0]) * coef + rooty = (0.955887 * dataxyz["head"][1] + 0.044113 * dataxyz["pelvis"][1]) * coef + rootz = (0.955887 * dataxyz["head"][2] + 0.044113 * dataxyz["pelvis"][2]) * coef + rootv = (0.955887 * dataxyz["head"][3] + 0.044113 * dataxyz["pelvis"][3]) + data["spine_01"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["spine_01"] = [rootx, rooty, rootz, rootv] + + rootx = (0.864201 * dataxyz["head"][0] + 0.135799 * dataxyz["pelvis"][0]) * coef + rooty = (0.864201 * dataxyz["head"][1] + 0.135799 * dataxyz["pelvis"][1]) * coef + rootz = (0.864201 * dataxyz["head"][2] + 0.135799 * dataxyz["pelvis"][2]) * coef + rootv = (0.864201 * dataxyz["head"][3] + 0.135799 * dataxyz["pelvis"][3]) + data["spine_02"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["spine_02"] = [rootx, rooty, rootz, rootv] + + rootx = (0.717787 * dataxyz["head"][0] + 0.282213 * dataxyz["pelvis"][0]) * coef + rooty = (0.717787 * dataxyz["head"][1] + 0.282213 * dataxyz["pelvis"][1]) * coef + rootz = (0.717787 * dataxyz["head"][2] + 0.282213 * dataxyz["pelvis"][2]) * coef + rootv = (0.717787 * dataxyz["head"][3] + 0.282213 * dataxyz["pelvis"][3]) + data["spine_03"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["spine_03"] = [rootx, rooty, rootz, rootv] + + rootx = (0.548406 * dataxyz["head"][0] + 0.451594 * dataxyz["pelvis"][0]) * coef + rooty = (0.548406 * dataxyz["head"][1] + 0.451594 * dataxyz["pelvis"][1]) * coef + rootz = (0.548406 * dataxyz["head"][2] + 0.451594 * dataxyz["pelvis"][2]) * coef + rootv = (0.548406 * dataxyz["head"][3] + 0.451594 * dataxyz["pelvis"][3]) + data["spine_04"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["spine_04"] = [rootx, rooty, rootz, rootv] + + rootx = (0.228975 * dataxyz["head"][0] + 0.771025 * dataxyz["pelvis"][0]) * coef + rooty = (0.228975 * dataxyz["head"][1] + 0.771025 * dataxyz["pelvis"][1]) * coef + rootz = (0.228975 * dataxyz["head"][2] + 0.771025 * dataxyz["pelvis"][2]) * coef + rootv = (0.228975 * dataxyz["head"][3] + 0.771025 * dataxyz["pelvis"][3]) + data["spine_05"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["spine_05"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[26].x * coef + rooty = -poseslms[26].z * coef + rootz = -poseslms[26].y * coef + rootv = poseslms[26].visibility + data["knee_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["knee_r"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[25].x * coef + rooty = -poseslms[25].z * coef + rootz = -poseslms[25].y * coef + rootv = poseslms[25].visibility + data["knee_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["knee_l"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[28].x * coef + rooty = -poseslms[28].z * coef + rootz = -poseslms[28].y * coef + rootv = poseslms[28].visibility + data["ankle_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["ankle_r"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[27].x * coef + rooty = -poseslms[27].z * coef + rootz = -poseslms[27].y * coef + rootv = poseslms[27].visibility + data["ankle_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["ankle_l"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[32].x * coef + rooty = -poseslms[32].z * coef + rootz = -poseslms[32].y * coef + rootv = poseslms[32].visibility + data["ball_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["ball_r"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[31].x * coef + rooty = -poseslms[31].z * coef + rootz = -poseslms[31].y * coef + rootv = poseslms[31].visibility + data["ball_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["ball_l"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[30].x * coef + rooty = -poseslms[30].z * coef + rootz = -poseslms[30].y * coef + rootv = poseslms[30].visibility + data["heel_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["heel_r"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[29].x * coef + rooty = -poseslms[29].z * coef + rootz = -poseslms[29].y * coef + rootv = poseslms[29].visibility + data["heel_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["heel_l"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[12].x * coef + rooty = -poseslms[12].z * coef + rootz = -poseslms[12].y * coef + rootv = poseslms[12].visibility + data["shoulder_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["shoulder_r"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[11].x * coef + rooty = -poseslms[11].z * coef + rootz = -poseslms[11].y * coef + rootv = poseslms[11].visibility + data["shoulder_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["shoulder_l"] = [rootx, rooty, rootz, rootv] + + rootx = (0.635 * dataxyz["shoulder_r"][0] + 0.365 * dataxyz["shoulder_l"][0]) * coef + rooty = 0.5 * (0.635 * dataxyz["shoulder_r"][1] + 0.365 * dataxyz["shoulder_l"][1] + 0.31 * dataxyz["head"][1] + 0.69 * dataxyz["spine_05"][1]) * coef + rootz = (0.0635 * dataxyz["shoulder_r"][1] + 0.0365 * dataxyz["shoulder_l"][1] + 0.9 * dataxyz["spine_05"][1]) * coef + rootv = (0.635 * dataxyz["shoulder_r"][0] + 0.451594 * dataxyz["shoulder_l"][0]) + data["clavicle_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["clavicle_r"] = [rootx, rooty, rootz, rootv] + + rootx = (0.635 * dataxyz["shoulder_l"][0] + 0.365 * dataxyz["shoulder_r"][0]) * coef + rooty = 0.5 * (0.635 * dataxyz["shoulder_l"][1] + 0.365 * dataxyz["shoulder_r"][1] + 0.31 * dataxyz["head"][1] + 0.69 * dataxyz["spine_05"][1]) * coef + rootz = (0.0635 * dataxyz["shoulder_l"][1] + 0.0365 * dataxyz["shoulder_r"][1] + 0.9 * dataxyz["spine_05"][1]) * coef + rootv = (0.635 * dataxyz["shoulder_l"][0] + 0.365 * dataxyz["shoulder_r"][0]) + data["clavicle_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["clavicle_l"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[14].x * coef + rooty = -poseslms[14].z * coef + rootz = -poseslms[14].y * coef + rootv = poseslms[14].visibility + data["elbow_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["elbow_r"] = [rootx, rooty, rootz, rootv] + + rootx = poseslms[13].x * coef + rooty = -poseslms[13].z * coef + rootz = -poseslms[13].y * coef + rootv = poseslms[13].visibility + data["elbow_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["elbow_l"] = [rootx, rooty, rootz, rootv] + +def rhandconvert(rhandlms, data, coef): + dataxyz = {} + + rootx = rhandlms[0].x * coef + rooty = -rhandlms[0].z * coef + rootz = -rhandlms[0].y * coef + rootv = 1.0 + data["hand_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["hand_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[5].x * coef + rooty = -rhandlms[5].z * coef + rootz = -rhandlms[5].y * coef + rootv = 1.0 + data["index_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["index_2_r"] = [rootx, rooty, rootz, rootv] + + rootx = (0.75 * dataxyz["hand_r"][0] + 0.25 * dataxyz["index_2_r"][0]) * coef + rooty = (0.75 * dataxyz["hand_r"][1] + 0.25 * dataxyz["index_2_r"][1]) * coef + rootz = (0.75 * dataxyz["hand_r"][2] + 0.25 * dataxyz["index_2_r"][2]) * coef + rootv = 1.0 + data["index_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["index_1_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[9].x * coef + rooty = -rhandlms[9].z * coef + rootz = -rhandlms[9].y * coef + rootv = 1.0 + data["middle_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["middle_2_r"] = [rootx, rooty, rootz, rootv] + + rootx = (0.75 * dataxyz["hand_r"][0] + 0.25 * dataxyz["middle_2_r"][0]) * coef + rooty = (0.75 * dataxyz["hand_r"][1] + 0.25 * dataxyz["middle_2_r"][1]) * coef + rootz = (0.75 * dataxyz["hand_r"][2] + 0.25 * dataxyz["middle_2_r"][2]) * coef + rootv = 1.0 + data["middle_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["middle_1_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[13].x * coef + rooty = -rhandlms[13].z * coef + rootz = -rhandlms[13].y * coef + rootv = 1.0 + data["ring_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["ring_2_r"] = [rootx, rooty, rootz, rootv] + + rootx = (0.75 * dataxyz["hand_r"][0] + 0.25 * dataxyz["ring_2_r"][0]) * coef + rooty = (0.75 * dataxyz["hand_r"][1] + 0.25 * dataxyz["ring_2_r"][1]) * coef + rootz = (0.75 * dataxyz["hand_r"][2] + 0.25 * dataxyz["ring_2_r"][2]) * coef + rootv = 1.0 + data["ring_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["ring_1_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[17].x * coef + rooty = -rhandlms[17].z * coef + rootz = -rhandlms[17].y * coef + rootv = 1.0 + data["pinky_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["pinky_2_r"] = [rootx, rooty, rootz, rootv] + + rootx = (0.75 * dataxyz["hand_r"][0] + 0.25 * dataxyz["pinky_2_r"][0]) * coef + rooty = (0.75 * dataxyz["hand_r"][1] + 0.25 * dataxyz["pinky_2_r"][1]) * coef + rootz = (0.75 * dataxyz["hand_r"][2] + 0.25 * dataxyz["pinky_2_r"][2]) * coef + rootv = 1.0 + data["pinky_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["pinky_1_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[1].x * coef + rooty = -rhandlms[1].z * coef + rootz = -rhandlms[1].y * coef + rootv = 1.0 + data["thumb_1_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["thumb_1_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[2].x * coef + rooty = -rhandlms[2].z * coef + rootz = -rhandlms[2].y * coef + rootv = 1.0 + data["thumb_2_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["thumb_2_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[3].x * coef + rooty = -rhandlms[3].z * coef + rootz = -rhandlms[3].y * coef + rootv = 1.0 + data["thumb_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["thumb_3_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[6].x * coef + rooty = -rhandlms[6].z * coef + rootz = -rhandlms[6].y * coef + rootv = 1.0 + data["index_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["index_3_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[7].x * coef + rooty = -rhandlms[7].z * coef + rootz = -rhandlms[7].y * coef + rootv = 1.0 + data["index_4_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["index_4_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[10].x * coef + rooty = -rhandlms[10].z * coef + rootz = -rhandlms[10].y * coef + rootv = 1.0 + data["middle_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["middle_3_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[11].x * coef + rooty = -rhandlms[11].z * coef + rootz = -rhandlms[11].y * coef + rootv = 1.0 + data["middle_4_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["middle_4_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[14].x * coef + rooty = -rhandlms[14].z * coef + rootz = -rhandlms[14].y * coef + rootv = 1.0 + data["ring_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["ring_3_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[15].x * coef + rooty = -rhandlms[15].z * coef + rootz = -rhandlms[15].y * coef + rootv = 1.0 + data["ring_4_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["ring_4_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[18].x * coef + rooty = -rhandlms[18].z * coef + rootz = -rhandlms[18].y * coef + rootv = 1.0 + data["pinky_3_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["pinky_3_r"] = [rootx, rooty, rootz, rootv] + + rootx = rhandlms[19].x * coef + rooty = -rhandlms[19].z * coef + rootz = -rhandlms[19].y * coef + rootv = 1.0 + data["pinky_4_r"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["pinky_4_r"] = [rootx, rooty, rootz, rootv] + +def lhandconvert(lhandlms, data, coef): + dataxyz = {} + + rootx = lhandlms[0].x * coef + rooty = -lhandlms[0].z * coef + rootz = -lhandlms[0].y * coef + rootv = 1.0 + data["hand_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["hand_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[5].x * coef + rooty = -lhandlms[5].z * coef + rootz = -lhandlms[5].y * coef + rootv = 1.0 + data["index_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["index_2_l"] = [rootx, rooty, rootz, rootv] + + rootx = (0.75 * dataxyz["hand_l"][0] + 0.25 * dataxyz["index_2_l"][0]) * coef + rooty = (0.75 * dataxyz["hand_l"][1] + 0.25 * dataxyz["index_2_l"][1]) * coef + rootz = (0.75 * dataxyz["hand_l"][2] + 0.25 * dataxyz["index_2_l"][2]) * coef + rootv = 1.0 + data["index_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["index_1_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[9].x * coef + rooty = -lhandlms[9].z * coef + rootz = -lhandlms[9].y * coef + rootv = 1.0 + data["middle_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["middle_2_l"] = [rootx, rooty, rootz, rootv] + + rootx = (0.75 * dataxyz["hand_l"][0] + 0.25 * dataxyz["middle_2_l"][0]) * coef + rooty = (0.75 * dataxyz["hand_l"][1] + 0.25 * dataxyz["middle_2_l"][1]) * coef + rootz = (0.75 * dataxyz["hand_l"][2] + 0.25 * dataxyz["middle_2_l"][2]) * coef + rootv = 1.0 + data["middle_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["middle_1_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[13].x * coef + rooty = -lhandlms[13].z * coef + rootz = -lhandlms[13].y * coef + rootv = 1.0 + data["ring_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["ring_2_l"] = [rootx, rooty, rootz, rootv] + + rootx = (0.75 * dataxyz["hand_l"][0] + 0.25 * dataxyz["ring_2_l"][0]) * coef + rooty = (0.75 * dataxyz["hand_l"][1] + 0.25 * dataxyz["ring_2_l"][1]) * coef + rootz = (0.75 * dataxyz["hand_l"][2] + 0.25 * dataxyz["ring_2_l"][2]) * coef + rootv = 1.0 + data["ring_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["ring_1_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[17].x * coef + rooty = -lhandlms[17].z * coef + rootz = -lhandlms[17].y * coef + rootv = 1.0 + data["pinky_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["pinky_2_l"] = [rootx, rooty, rootz, rootv] + + rootx = (0.75 * dataxyz["hand_l"][0] + 0.25 * dataxyz["pinky_2_l"][0]) * coef + rooty = (0.75 * dataxyz["hand_l"][1] + 0.25 * dataxyz["pinky_2_l"][1]) * coef + rootz = (0.75 * dataxyz["hand_l"][2] + 0.25 * dataxyz["pinky_2_l"][2]) * coef + rootv = 1.0 + data["pinky_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["pinky_1_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[1].x * coef + rooty = -lhandlms[1].z * coef + rootz = -lhandlms[1].y * coef + rootv = 1.0 + data["thumb_1_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["thumb_1_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[2].x * coef + rooty = -lhandlms[2].z * coef + rootz = -lhandlms[2].y * coef + rootv = 1.0 + data["thumb_2_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["thumb_2_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[3].x * coef + rooty = -lhandlms[3].z * coef + rootz = -lhandlms[3].y * coef + rootv = 1.0 + data["thumb_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["thumb_3_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[6].x * coef + rooty = -lhandlms[6].z * coef + rootz = -lhandlms[6].y * coef + rootv = 1.0 + data["index_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["index_3_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[7].x * coef + rooty = -lhandlms[7].z * coef + rootz = -lhandlms[7].y * coef + rootv = 1.0 + data["index_4_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["index_4_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[10].x * coef + rooty = -lhandlms[10].z * coef + rootz = -lhandlms[10].y * coef + rootv = 1.0 + data["middle_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["middle_3_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[11].x * coef + rooty = -lhandlms[11].z * coef + rootz = -lhandlms[11].y * coef + rootv = 1.0 + data["middle_4_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["middle_4_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[14].x * coef + rooty = -lhandlms[14].z * coef + rootz = -lhandlms[14].y * coef + rootv = 1.0 + data["ring_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["ring_3_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[15].x * coef + rooty = -lhandlms[15].z * coef + rootz = -lhandlms[15].y * coef + rootv = 1.0 + data["ring_4_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["ring_4_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[18].x * coef + rooty = -lhandlms[18].z * coef + rootz = -lhandlms[18].y * coef + rootv = 1.0 + data["pinky_3_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["pinky_3_l"] = [rootx, rooty, rootz, rootv] + + rootx = lhandlms[19].x * coef + rooty = -lhandlms[19].z * coef + rootz = -lhandlms[19].y * coef + rootv = 1.0 + data["pinky_4_l"] = makexyzrotvis(rootx, rooty, rootz, 0, 0, 0, rootv) + dataxyz["pinky_4_l"] = [rootx, rooty, rootz, rootv] diff --git a/README.md b/README.md new file mode 100644 index 0000000..1a3817d --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ +# Human Pose Estimation with mediapipe + +## Запуск + +Для вебкамеры, где 0 - индекс самой вебкамеры, если отличается, то следует заменить на нужный. +```console +python3 hpe_videocapture.py 0 scale_pose +``` + +Для чтения из файла +```console +python3 hpe_videocapture.py path_to_video scale_pose +``` + +Параметр scale_pose - ширина плеч. По умолчанию равен 0,42. + +Есть экспериментальная версия для взаимодействия с ffmpeg и osc: +```console +python3 hpe_ffmpeg.py +``` +Внутри кода записаны параметры для osc соединения: +* osc_address = "10.199.0.14" +* osc_port = 5005 + +Предполагается вывести выше указанные параметры либо в аргументы, либо в конфиг-файл. + +## Зависимости + +Библиотеки python: +* mediapipe +* numpy +* matplotlib (можно сделать опциональной) +* opencv-python +* json +* python-osc (в файле hpe_ffmpeg.py) + +Библиотека mediapipe требует cudatoolkit и cudnn для работы с gpu. + +## Интерфейс для hpe_videocapture + +Для разовой отрисовки точек в 3D с помощью matplotlib необходимо нажать клавишу 'm'. + +Чтобы записать сконвертированные данные в модель UE4 в файл 'hierarchy_data.json', нужно нажать клавишу 'j'. + +Для закрытия программы нужно нажать клавишу 'esc'. + +## Требования к постановке эксперимента + +* Человек в кадре должен быть один. +* Камера примерно должна быть под углом 90 градусов. +* Человек должен входить в кадр целиком. +* Желательно, чтобы у одежда на человеке были хорошо видны контуры. + +## Сборка на Windows + +В разработке + +## Config file + +В разработке. + +Предположительные переменные. + +Параметры класса hpe_mp_class из hpe_mp_class.py: +* hands_static_image_mode = False +* hands_max_num_hands = 2 +* hands_min_detection_confidence = 0.7 +* hands_min_tracking_confidence = 0.5 +* pose_static_image_mode = False +* pose_upper_body_only = False +* pose_smooth_landmarks = True +* pose_min_detection_confidence = 0.7 +* pose_min_tracking_confidence = 0.5 +* hol_static_image_mode = False +* hol_upper_body_only = False +* hol_smooth_landmarks = True +* hol_min_detection_confidence = 0.7 +* hol_min_tracking_confidence = 0.5 + +Физические параметры человека (ширина плеч в метрах): +* scale_pose = 0,42 diff --git a/hpe_ffmpeg.py b/hpe_ffmpeg.py new file mode 100644 index 0000000..fb94963 --- /dev/null +++ b/hpe_ffmpeg.py @@ -0,0 +1,66 @@ +import sys +import cv2 +import time +from hpe_mp_class import hpe_mp_class +from pythonosc import udp_client + +# Arguments +adress_input = "in.pipe1" +if len(sys.argv) > 1: + adress_input = sys.argv[1] + for i in range(0, 100): + if adress_input == str(i): + adress_input = i + +osc_address = "10.199.0.14" +osc_port = 5005 +client = udp_client.SimpleUDPClient(osc_address, osc_port) + +showing = False + +# Videocapture +cap = cv2.VideoCapture(adress_input) + +# Preprocessing parameters +crop = 1.0 +frame_width = int(crop*cap.get(cv2.CAP_PROP_FRAME_WIDTH)) +frame_height = int(crop*cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + +# FPS variables +pTime = 0 +cTime = 0 + +# Mediapie class +mp_cl = hpe_mp_class() + +while True: + # Reading frame + success, img = cap.read() + + # Image preprocessing + img = cv2.resize(img, (frame_width, frame_height)) + + # Mediapipe + mp_cl.process(img) + mp_cl.show(img) + + if showing: + # FPS + cTime = time.time() + fps = 1. / (cTime - pTime) + pTime = cTime + + # Showing + img = cv2.flip(img, 1) # mirror + cv2.putText(img, str(int(fps)), (22, 32), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2) + cv2.putText(img, str(int(fps)), (20, 30), cv2.FONT_HERSHEY_COMPLEX, 1, (255, 255, 255), 2) + cv2.imshow("Main", img) + + # get converted map + res = mp_cl.getJSON() + client.send_message("/pose/0", res) + + # Interface + key = cv2.waitKey(1) + if key == 27: + break diff --git a/hpe_mp_class.py b/hpe_mp_class.py new file mode 100644 index 0000000..4f70351 --- /dev/null +++ b/hpe_mp_class.py @@ -0,0 +1,204 @@ +import cv2 +import mediapipe as mp +import numpy as np + +mpHands = mp.solutions.hands +mpPose = mp.solutions.pose +mpDraw = mp.solutions.drawing_utils +mpHolistic = mp.solutions.holistic + +import logging +FORMAT = '%(asctime)-15s %(message)s' +logging.basicConfig(filename="hpe_mp_class.log", level=logging.INFO, format=FORMAT) +logger = logging.getLogger("hpe_mp_class_logger") + +from ModelUE4 import bodyconvert +from ModelUE4 import rhandconvert +from ModelUE4 import lhandconvert + +class hpe_mp_class(): + + def __init__(self): + hands_static_image_mode = False + hands_max_num_hands = 2 + hands_min_detection_confidence = 0.7 + hands_min_tracking_confidence = 0.5 + + pose_static_image_mode = False + pose_upper_body_only = False + pose_smooth_landmarks = True + pose_min_detection_confidence = 0.7 + pose_min_tracking_confidence = 0.5 + + hol_static_image_mode = False + hol_upper_body_only = False + hol_smooth_landmarks = True + hol_min_detection_confidence = 0.7 + hol_min_tracking_confidence = 0.5 + + self.holistic_use = True + self.coef = 1.0 + + try: + self.hands = mpHands.Hands(hands_static_image_mode, + hands_max_num_hands, + hands_min_detection_confidence, + hands_min_tracking_confidence) + self.poses = mpPose.Pose(pose_static_image_mode, + pose_upper_body_only, + pose_smooth_landmarks, + pose_min_detection_confidence, + pose_min_tracking_confidence) + self.hol = mpHolistic.Holistic(hol_static_image_mode, + hol_upper_body_only, + hol_smooth_landmarks, + hol_min_detection_confidence, + hol_min_tracking_confidence) + logger.info("Success initialize hpe class") + except Exception as err: + logger.exception("Error initialize hpe class: " + str(err)) + + def process(self, image, scale_pose=0.42): + try: + imgRGB = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + + if self.holistic_use: + self.results_hol = self.hol.process(image) + + poseslms = {} + if self.results_hol.pose_landmarks: + for id, lm in enumerate(self.results_hol.pose_landmarks.landmark): + poseslms[id] = lm + rast = np.sqrt(pow(poseslms[11].x - poseslms[12].x, 2) + pow(poseslms[11].y - poseslms[12].y, 2) + pow(poseslms[11].z - poseslms[12].z, 2)) + self.coef = scale_pose / rast + else: + self.results_hands = self.hands.process(imgRGB) + self.results_pose = self.poses.process(imgRGB) + except Exception as err: + logger.exception("Error processing hpe class: " + str(err)) + + def show(self, image): + try: + if self.holistic_use: + mpDraw.draw_landmarks( + image, + self.results_hol.face_landmarks, + mpHolistic.FACE_CONNECTIONS) + mpDraw.draw_landmarks( + image, + self.results_hol.pose_landmarks, + mpHolistic.POSE_CONNECTIONS) + mpDraw.draw_landmarks( + image, + self.results_hol.left_hand_landmarks, + mpHands.HAND_CONNECTIONS) + mpDraw.draw_landmarks( + image, + self.results_hol.right_hand_landmarks, + mpHands.HAND_CONNECTIONS) + else: + if self.results_hands.multi_hand_landmarks: + for handLms in self.results_hands.multi_hand_landmarks: + mpDraw.draw_landmarks(image, handLms, mpHands.HAND_CONNECTIONS) + for id, lm in enumerate(handLms.landmark): + h, w, c = image.shape + cx, cy = int(lm.x * w), int(lm.y * h) + if id == 4 or id == 8 or id == 12 or id == 16 or id == 20: + cv2.circle(image, (cx, cy), 10, (255, 0, 0), cv2.FILLED) + + if self.results_pose.pose_landmarks: + mpDraw.draw_landmarks(image, self.results_pose.pose_landmarks, mpPose.POSE_CONNECTIONS) + except Exception as err: + logger.exception("Error showing hpe class: " + str(err)) + + def getResults(self): + res = {} + try: + poseslms = {} + handslms = [] + + if self.holistic_use: + if self.results_hol.pose_landmarks: + for id, lm in enumerate(self.results_hol.pose_landmarks.landmark): + poseslms[id] = lm + hand = {} + if self.results_hol.left_hand_landmarks: + for id, lm in enumerate(self.results_hol.left_hand_landmarks.landmark): + hand[id] = lm + handslms.append(hand) + if self.results_hol.right_hand_landmarks: + for id, lm in enumerate(self.results_hol.right_hand_landmarks.landmark): + hand[id+21] = lm + handslms.append(hand) + res["poses"] = poseslms + res["hands"] = handslms + else: + if self.results_pose.pose_landmarks: + for id, lm in enumerate(self.results_pose.pose_landmarks.landmark): + poseslms[id] = lm + if self.results_hands.multi_hand_landmarks: + for handLms in self.results_hands.multi_hand_landmarks: + hand = {} + for id, lm in enumerate(handLms.landmark): + hand[id] = lm + handslms.append(hand) + res["poses"] = poseslms + res["hands"] = handslms + except Exception as err: + logger.exception("Error getting result hpe class: " + str(err)) + + return res + + def scaleResult(self, result): + try: + if self.holistic_use: + for pp in result["poses"].keys(): + result["poses"][pp].x *= self.coef + result["poses"][pp].y *= self.coef + result["poses"][pp].z *= self.coef + + if len(result["hands"]) > 0: + for hp0 in result["hands"][0].keys(): + result["hands"][0][hp0].x *= self.coef + result["hands"][0][hp0].y *= self.coef + result["hands"][0][hp0].z *= self.coef + + if len(result["hands"]) > 1: + for hp1 in result["hands"][1].keys(): + result["hands"][1][hp1].x *= self.coef + result["hands"][1][hp1].y *= self.coef + result["hands"][1][hp1].z *= self.coef + except Exception as err: + logger.exception("Error scaling hpe class: " + str(err)) + + def getJSON(self): + data = {} + try: + if self.holistic_use: + poseslms = {} + maxy = 0 + if self.results_hol.pose_landmarks: + for id, lm in enumerate(self.results_hol.pose_landmarks.landmark): + poseslms[id] = lm + if lm.y > maxy: + maxy = lm.y + + bodyconvert(poseslms, data, self.coef, maxy) + + rhandlms = {} + if self.results_hol.right_hand_landmarks: + for id, lm in enumerate(self.results_hol.right_hand_landmarks.landmark): + rhandlms[id] = lm + + rhandconvert(rhandlms, data, self.coef) + + lhandlms = {} + if self.results_hol.left_hand_landmarks: + for id, lm in enumerate(self.results_hol.left_hand_landmarks.landmark): + lhandlms[id] = lm + + lhandconvert(lhandlms, data, self.coef) + except Exception as err: + logger.exception("Error json converting hpe class: " + str(err)) + + return data \ No newline at end of file diff --git a/hpe_videocapture.py b/hpe_videocapture.py new file mode 100644 index 0000000..cbec62b --- /dev/null +++ b/hpe_videocapture.py @@ -0,0 +1,90 @@ +import sys +import cv2 +import time +from hpe_mp_class import hpe_mp_class +import matplotlib.pyplot as plt +import json + +# Arguments +adress_input = sys.argv[1] +for i in range(0, 100): + if adress_input == str(i): + adress_input = i +scale_pose = 0.42 +if len(sys.argv) > 2: + scale_pose = float(sys.argv[2]) + +# Videocapture +cap = cv2.VideoCapture(adress_input) + +# Preprocessing parameters +crop = 1.0 +frame_width = int(crop*cap.get(cv2.CAP_PROP_FRAME_WIDTH)) +frame_height = int(crop*cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + +# FPS variables +pTime = 0 +cTime = 0 + +# Mediapie class +mp_cl = hpe_mp_class() + +while True: + # Reading frame + success, img = cap.read() + + # Image preprocessing + img = cv2.resize(img, (frame_width, frame_height)) + + # Mediapipe + mp_cl.process(img, scale_pose=scale_pose) + mp_cl.show(img) + + # FPS + cTime = time.time() + fps = 1. / (cTime - pTime) + pTime = cTime + + # Showing + img = cv2.flip(img, 1) # mirror + cv2.putText(img, str(int(fps)), (22, 32), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2) + cv2.putText(img, str(int(fps)), (20, 30), cv2.FONT_HERSHEY_COMPLEX, 1, (255, 255, 255), 2) + cv2.imshow("Main", img) + + # Interface + key = cv2.waitKey(1) + if key == 27: + break + if key == 109: + # Matplotlib + fig = plt.figure() + ax = fig.add_subplot(111, projection='3d') + ax.set_xlabel('X') + ax.set_ylabel('Y') + ax.set_zlabel('Z') + res = mp_cl.getResults() + mp_cl.scaleResult(res) + points_body_x = [] + points_body_y = [] + points_body_z = [] + points_hand_x = [] + points_hand_y = [] + points_hand_z = [] + for pp in res["poses"].values(): + if pp.visibility > 0.9: + points_body_x.append(pp.x) + points_body_y.append(pp.y) + points_body_z.append(pp.z) + for hand in res["hands"]: + for hp in hand.values(): + points_hand_x.append(hp.x) + points_hand_y.append(hp.y) + points_hand_z.append(hp.z) + ax.scatter(points_body_x, points_body_y, points_body_z, color='blue') + ax.scatter(points_hand_x, points_hand_y, points_hand_z, color='green') + plt.show() + if key == 106: + # JSON + res = mp_cl.getJSON() + with open('hierarchy_data.json', 'w', encoding='utf-8') as f: + json.dump(res, f, ensure_ascii=False, indent=4)