MediaPipeとUnityをUDPで繋ぐ　-その4-

アバター

はじめに

前回までは、顔のオブジェクトはUnityで作っていましたが、今回はBlenderで顔を作ってBlendShapeを適用してみました☺️
手はLandmarkを繋ぐように、Sphereを引き延ばして手の輪郭を作りました。
本当は手もモデルを使いたかったのですが、指の回転が難しく🥺

Python

前回から変更なし

Blender

顔のモデルを作成しました。
パーツ構成：
　顔
　目(左右)
　瞳(左右)
　眉(左右)
口と瞼は、顔のメッシュ変形で表現します。


怖いて😂
動作がわかる形なら何でも良いやーと、思いつきで作っていたらこうなりました。
どうやらeightにはデザインセンスがなかったようです🤔
手にはアーマチュアも仕込んでありますが、今回使うのは顔だけです。

シェイプキーはこれらを登録しました。
顔オブジェクトのシェイプキー


眉オブジェクトのシェイプキー




BlendShapeで反映されるのはメッシュの変形のみです。座標の移動や回転は反映されません。
そのため、たとえば瞬きであれば、瞼のメッシュを下方向へ引き伸ばすことで表現しています。

作成したら、FBX形式でエクスポートします。

Unity

もともとSceneにあった顔パーツは削除し、Blenderで作成したFBXをインポートして顔として使用します。

スクリプト

MediaPipe管理

前回から変更なし

UDP管理

前回から変更なし

顔パーツ管理

眉・瞼・口はBlendShapeを使用します。

using System.Collections.Generic;
using UnityEngine;

namespace PartsData_face_ns{

[System.Serializable]
public class PartsData_face
{
    public string parts = "";
    public float[] nose = new float[3];
    public float[] top = new float[3];
    public float[] bottom = new float[3];
    public float[] right = new float[3];
    public float[] left = new float[3];
    public float[] blendshape = new float[52];
}

public class FaceManager : MonoBehaviour
{
    public enum FaceBlendshapeName
    {
        Neutral = 0,
        BrowDownLeft,
        BrowDownRight,
        BrowInnerUp,
        BrowOuterUpLeft,
        BrowOuterUpRight,
        CheekPuff,
        CheekSquintLeft,
        CheekSquintRight,
        EyeBlinkLeft,
        EyeBlinkRight,
        EyeLookDownLeft,
        EyeLookDownRight,
        EyeLookInLeft,
        EyeLookInRight,
        EyeLookOutLeft,
        EyeLookOutRight,
        EyeLookUpLeft,
        EyeLookUpRight,
        EyeSquintLeft,
        EyeSquintRight,
        EyeWideLeft,
        EyeWideRight,
        JawForward,
        JawLeft,
        JawOpen,
        JawRight,
        MouthClose,
        MouthDimpleLeft,
        MouthDimpleRight,
        MouthFrownLeft,
        MouthFrownRight,
        MouthFunnel,
        MouthLeft,
        MouthLowerDownLeft,
        MouthLowerDownRight,
        MouthPressLeft,
        MouthPressRight,
        MouthPucker,
        MouthRight,
        MouthRollLower,
        MouthRollUpper,
        MouthShrugLower,
        MouthShrugUpper,
        MouthSmileLeft,
        MouthSmileRight,
        MouthStretchLeft,
        MouthStretchRight,
        MouthUpperUpLeft,
        MouthUpperUpRight,
        NoseSneerLeft,
        NoseSneerRight,
        FaceBlendshapeName_Max
    }

    public static Queue<string> rcvData_face;
    public Queue<PartsData_face> moveData_face;
    [SerializeField] GameObject RightBrow;
    [SerializeField] GameObject LeftBrow;
    [SerializeField] GameObject RightEye;
    [SerializeField] GameObject LeftEye;
    [SerializeField] GameObject RightPupil;
    [SerializeField] GameObject LeftPupil;
    [SerializeField] GameObject Nose;

    private int hidecnt;
    private Vector3 RightEye_angle_base;
    private Vector3 LeftEye_angle_base;
    private Vector3 Nose_pos_base;
    private Vector3 Nose_angle_base;

    private PartsData_face prev_moveData_face;
    private int rcvdata_devide; //受信データを分割して滑らかにする

    void Start()
    {
        hidecnt = 0;
        rcvdata_devide = 2;
        rcvData_face = new Queue<string>();
        moveData_face = new Queue<PartsData_face>();
        prev_moveData_face = null;
        ChangeVisibility(false);
        RightEye_angle_base = RightEye.transform.localEulerAngles;
        LeftEye_angle_base = LeftEye.transform.localEulerAngles;
        Nose_pos_base = Nose.transform.localPosition;
        Nose_angle_base = Nose.transform.localEulerAngles;
    }

    void Update()
    {
        hidecnt++;
        if((rcvData_face != null) && (rcvData_face.Count != 0))
        {
            string rcvdata = rcvData_face.Dequeue();
            PartsData_face rcv_face = JsonUtility.FromJson<PartsData_face>(rcvdata);

            //今回値を前回値の差を、今回値(目標)に向かって数回に分けて移動させる
            if(prev_moveData_face != null)
            {
                PartsData_face div_face = new PartsData_face();
                
                //前回値を開始地点とする
                div_face.parts = prev_moveData_face.parts;
                for(int idx=0; idx<3; idx++)
                {
                    div_face.nose[idx] = prev_moveData_face.nose[idx];
                    div_face.top[idx] = prev_moveData_face.top[idx];
                    div_face.bottom[idx] = prev_moveData_face.bottom[idx];
                    div_face.right[idx] = prev_moveData_face.right[idx];
                    div_face.left[idx] = prev_moveData_face.left[idx];
                }
                for(int idx=(int)FaceBlendshapeName.Neutral; idx<(int)FaceBlendshapeName.FaceBlendshapeName_Max; idx++)
                {
                    div_face.blendshape[idx] = prev_moveData_face.blendshape[idx];
                }

                //今回値と前回値の差分を分割し、キューへ格納する
                for(int div=0; div<rcvdata_devide-1; div++)
                {
                    for(int idx=0; idx<3; idx++)
                    {
                        div_face.nose[idx] += (rcv_face.nose[idx] - prev_moveData_face.nose[idx]) / rcvdata_devide;
                        div_face.top[idx] += (rcv_face.top[idx] - prev_moveData_face.top[idx]) / rcvdata_devide;
                        div_face.bottom[idx] += (rcv_face.bottom[idx] - prev_moveData_face.bottom[idx]) / rcvdata_devide;
                        div_face.right[idx] += (rcv_face.right[idx] - prev_moveData_face.right[idx]) / rcvdata_devide;
                        div_face.left[idx] += (rcv_face.left[idx] - prev_moveData_face.left[idx]) / rcvdata_devide;
                    }
                    for(int idx=(int)FaceBlendshapeName.Neutral; idx<(int)FaceBlendshapeName.FaceBlendshapeName_Max; idx++)
                    {
                        div_face.blendshape[idx] += (rcv_face.blendshape[idx] - prev_moveData_face.blendshape[idx]) / rcvdata_devide;
                    }
                    moveData_face.Enqueue(div_face);
                }
            }
            else
            {
                //ここを実行するのは初回のみ
                prev_moveData_face = new PartsData_face();
            }
            //分割数によっては最終値が中途半端になるため、最後は今回値ぴったりで終わらせる
            moveData_face.Enqueue(rcv_face);
            
            //今回値を前回値として保持
            prev_moveData_face.parts = rcv_face.parts;
            for(int idx=0; idx<3; idx++)
            {
                prev_moveData_face.nose[idx] = rcv_face.nose[idx];
                prev_moveData_face.top[idx] = rcv_face.top[idx];
                prev_moveData_face.bottom[idx] = rcv_face.bottom[idx];
                prev_moveData_face.right[idx] = rcv_face.right[idx];
                prev_moveData_face.left[idx] = rcv_face.left[idx];
            }
            for(int idx=(int)FaceBlendshapeName.Neutral; idx<(int)FaceBlendshapeName.FaceBlendshapeName_Max; idx++)
            {
                prev_moveData_face.blendshape[idx] = rcv_face.blendshape[idx];
            }
        }

        if((moveData_face != null) && (moveData_face.Count != 0))
        {
            PartsData_face face = moveData_face.Dequeue();

            hidecnt = 0;
            //処理順は固定
            NoseMove(face);
            BrowMove_blendshape(face);
            EyeBlink_blendshape(face);
            EyeMove(face);
            MouthMove_blendshape(face);
            ChangeVisibility(true);
        }
        if(hidecnt > 100)
        {
            hidecnt = 0;
            for(int i=(int)FaceBlendshapeName.Neutral; i<(int)FaceBlendshapeName.FaceBlendshapeName_Max; i++)
            {
                ChangeVisibility(false);
            }
            rcvData_face.Clear();
            moveData_face.Clear();
        }
    }

    private void OnApplicationQuit()
    {
        rcvData_face.Clear();
        moveData_face.Clear();
    }

    private void OnDestroy()
    {
        rcvData_face.Clear();
        moveData_face.Clear();
    }

    public static void SetRcvData(string rcvdata)
    {
        rcvData_face.Enqueue(rcvdata);
    }

    private void NoseMove(PartsData_face face)
    {
        Transform Nose_Transform = Nose.transform;

        //鼻の位置を計算する
        Vector3 Nose_pos = Nose_Transform.localPosition;
        Nose_pos.x = Nose_pos_base.x + ((face.nose[0]*6f) * (-1f));
        Nose_pos.y = Nose_pos_base.y + ((face.nose[1]*3f) * (-1f));
        Nose_pos.z = Nose_pos_base.z + (face.nose[2]*100f);
        Nose_Transform.localPosition = Nose_pos;

        //顔の上下左右端の座標から、顔全体の傾きを計算する
        Vector3 Nose_angle = Nose_Transform.localEulerAngles;
        Nose_angle.x = Nose_angle_base.x + ((face.top[2] - face.bottom[2]) * 250f);
        Nose_angle.y = Nose_angle_base.y + ((face.right[2] - face.left[2]) * 250f);
        Nose_angle.z = Nose_angle_base.z + ((face.top[0] - face.bottom[0]) * 250f);
        Nose_Transform.localEulerAngles = Nose_angle;
    }

    private void BrowMove_blendshape(PartsData_face face)
    {
        SkinnedMeshRenderer rightbrow_smr = RightBrow.GetComponent<SkinnedMeshRenderer>();
        SkinnedMeshRenderer leftbrow_smr = LeftBrow.GetComponent<SkinnedMeshRenderer>();
        
        float rightbrow = (face.blendshape[(int)FaceBlendshapeName.BrowInnerUp])*120;
        float leftbrow = (face.blendshape[(int)FaceBlendshapeName.BrowInnerUp])*120;
        if(rightbrow > 100.0f) rightbrow = 100.0f;
        if(leftbrow > 100.0f) leftbrow = 100.0f;

        rightbrow_smr.SetBlendShapeWeight(0, rightbrow);
        leftbrow_smr.SetBlendShapeWeight(0, leftbrow);
    }

    private void EyeBlink_blendshape(PartsData_face face)
    {
        SkinnedMeshRenderer Nose_smr;
        Nose_smr = Nose.GetComponent<SkinnedMeshRenderer>();

        float rightblink = (face.blendshape[(int)FaceBlendshapeName.EyeBlinkRight])*120;
        float leftblink = (face.blendshape[(int)FaceBlendshapeName.EyeBlinkLeft])*120;
        if(rightblink > 100.0f) rightblink = 100.0f;
        if(leftblink > 100.0f) leftblink = 100.0f;

        Nose_smr.SetBlendShapeWeight(2, rightblink);
        Nose_smr.SetBlendShapeWeight(3, leftblink);
    }

    private void EyeMove(PartsData_face face)
    {
        EyeMove_Right(face);
        EyeMove_Left(face);
    }

    private void EyeMove_Right(PartsData_face face)
    {
        float eyemove_data_updown;
        float eyemove_data_inout;

        //上下方向に大きい値の方向へ動かす
        if(face.blendshape[(int)FaceBlendshapeName.EyeLookDownRight] > face.blendshape[(int)FaceBlendshapeName.EyeLookUpRight])
        {
            eyemove_data_updown = face.blendshape[(int)FaceBlendshapeName.EyeLookDownRight] * (-1f);
        }
        else
        {
            eyemove_data_updown = face.blendshape[(int)FaceBlendshapeName.EyeLookUpRight];
        }

        //左右方向に大きい値の方向へ動かす
        if(face.blendshape[(int)FaceBlendshapeName.EyeLookInRight] > face.blendshape[(int)FaceBlendshapeName.EyeLookOutRight])
        {
            eyemove_data_inout = face.blendshape[(int)FaceBlendshapeName.EyeLookInRight] * (-1f);
        }
        else
        {
            eyemove_data_inout = face.blendshape[(int)FaceBlendshapeName.EyeLookOutRight];
        }

        Transform RightEye_Transform = RightEye.transform;
        Vector3 RightEye_angle = RightEye_Transform.localEulerAngles;
        RightEye_angle.x = RightEye_angle_base.x + (eyemove_data_updown*25); //上下方向の回転はx軸を中心とする
        RightEye_angle.y = RightEye_angle_base.y + ((eyemove_data_inout*25) * (-1)); //左右方向の回転はy軸を中心とする
        RightEye_Transform.localEulerAngles = RightEye_angle;
    }

    private void EyeMove_Left(PartsData_face face)
    {
        float eyemove_data_updown;
        float eyemove_data_inout;

        //上下方向に大きい値の方向へ動かす
        if(face.blendshape[(int)FaceBlendshapeName.EyeLookDownLeft] > face.blendshape[(int)FaceBlendshapeName.EyeLookUpLeft])
        {
            eyemove_data_updown = face.blendshape[(int)FaceBlendshapeName.EyeLookDownLeft] * (-1f);
        }
        else
        {
            eyemove_data_updown = face.blendshape[(int)FaceBlendshapeName.EyeLookUpLeft];
        }

        //左右方向に大きい値の方向へ動かす
        if(face.blendshape[(int)FaceBlendshapeName.EyeLookInLeft] > face.blendshape[(int)FaceBlendshapeName.EyeLookOutLeft])
        {
            eyemove_data_inout = face.blendshape[(int)FaceBlendshapeName.EyeLookInLeft];
        }
        else
        {
            eyemove_data_inout = face.blendshape[(int)FaceBlendshapeName.EyeLookOutLeft] * (-1f);
        }

        Transform LeftEye_Transform = LeftEye.transform;
        Vector3 LeftEye_angle = LeftEye_Transform.localEulerAngles;
        LeftEye_angle.x = LeftEye_angle_base.x + (eyemove_data_updown*25); //上下方向の回転はx軸を中心とする
        LeftEye_angle.y = LeftEye_angle_base.y + ((eyemove_data_inout*25) * (-1)); //左右方向の回転はy軸を中心とする
        LeftEye_Transform.localEulerAngles = LeftEye_angle;
    }

    private void MouthMove_blendshape(PartsData_face face)
    {
        SkinnedMeshRenderer Nose_smr;
        Nose_smr = Nose.GetComponent<SkinnedMeshRenderer>();

        float jowopen = (face.blendshape[(int)FaceBlendshapeName.JawOpen])*120f;
        float mouthstretch = (face.blendshape[(int)FaceBlendshapeName.MouthStretchRight])*10000f;
        if(jowopen > 100.0f) jowopen = 100.0f;
        if(mouthstretch > 100.0f) mouthstretch = 100.0f;

        Nose_smr.SetBlendShapeWeight(0, jowopen);
        Nose_smr.SetBlendShapeWeight(1, mouthstretch);
    }

    private void ChangeVisibility(bool vis)
    {
        RightBrow.GetComponent<Renderer>().enabled = vis;
        LeftBrow.GetComponent<Renderer>().enabled = vis;
        RightEye.GetComponent<Renderer>().enabled = vis;
        LeftEye.GetComponent<Renderer>().enabled = vis;
        RightPupil.GetComponent<Renderer>().enabled = vis;
        LeftPupil.GetComponent<Renderer>().enabled = vis;
        Nose.GetComponent<Renderer>().enabled = vis;
    }
}
}

BlendShape使用箇所を見ていきます。

    private void BrowMove_blendshape(PartsData_face face)
    {
        SkinnedMeshRenderer rightbrow_smr = RightBrow.GetComponent<SkinnedMeshRenderer>();
        SkinnedMeshRenderer leftbrow_smr = LeftBrow.GetComponent<SkinnedMeshRenderer>();
        
        float rightbrow = (face.blendshape[(int)FaceBlendshapeName.BrowInnerUp])*120;
        float leftbrow = (face.blendshape[(int)FaceBlendshapeName.BrowInnerUp])*120;
        if(rightbrow > 100.0f) rightbrow = 100.0f;
        if(leftbrow > 100.0f) leftbrow = 100.0f;

        rightbrow_smr.SetBlendShapeWeight(0, rightbrow);
        leftbrow_smr.SetBlendShapeWeight(0, leftbrow);
    }

眉の制御です。
SkinnedMeshRendererのSetBlendShapeWeight()で、BlendShapeを動かします。
第1引数は、眉に設定したBlendShapeに割り当てられたIDを指定します。
第2引数は値です。MediaPipeからの受信データにBlendShapeの値を含めているので、それを参照します。
値は0〜100の範囲なので、100でリミットしています。

    private void EyeBlink_blendshape(PartsData_face face)
    {
        SkinnedMeshRenderer Nose_smr;
        Nose_smr = Nose.GetComponent<SkinnedMeshRenderer>();

        float rightblink = (face.blendshape[(int)FaceBlendshapeName.EyeBlinkRight])*120;
        float leftblink = (face.blendshape[(int)FaceBlendshapeName.EyeBlinkLeft])*120;
        if(rightblink > 100.0f) rightblink = 100.0f;
        if(leftblink > 100.0f) leftblink = 100.0f;

        Nose_smr.SetBlendShapeWeight(2, rightblink);
        Nose_smr.SetBlendShapeWeight(3, leftblink);
    }

瞬きの制御です。眉と同じですね。

    private void MouthMove_blendshape(PartsData_face face)
    {
        SkinnedMeshRenderer Nose_smr;
        Nose_smr = Nose.GetComponent<SkinnedMeshRenderer>();

        float jowopen = (face.blendshape[(int)FaceBlendshapeName.JawOpen])*120f;
        float mouthstretch = (face.blendshape[(int)FaceBlendshapeName.MouthStretchRight])*10000f;
        if(jowopen > 100.0f) jowopen = 100.0f;
        if(mouthstretch > 100.0f) mouthstretch = 100.0f;

        Nose_smr.SetBlendShapeWeight(0, jowopen);
        Nose_smr.SetBlendShapeWeight(1, mouthstretch);
    }

口の制御です。
stretch(横方向)の変化量は非常に小さいようなので、係数を大きくしました。

右手パーツ管理

using System.Collections.Generic;
using UnityEngine;

namespace PartsData_righthand_ns{

[System.Serializable]
public class PartsData_righthand
{
    public string parts = "";
    public float[] x = new float[21];
    public float[] y = new float[21];
    public float[] z = new float[21];
}

public class RightHandManager : MonoBehaviour
{
    public enum RightHandLandmarkName
    {
        Wrist = 0,
        Thumb_Cmc,
        Thumb_Mcp,
        Thumb_Ip,
        Thumb_Tip,
        Index_Finger_Mcp,
        Index_Finger_Pip,
        Index_Finger_Dip,
        Index_Finger_Tip,
        Middle_Finger_Mcp,
        Middle_Finger_Pip,
        Middle_Finger_Dip,
        Middle_Finger_Tip,
        Ring_Finger_Mcp,
        Ring_Finger_Pip,
        Ring_Finger_Dip,
        Ring_Finger_Tip,
        Pinky_Finger_Mcp,
        Pinky_Finger_Pip,
        Pinky_Finger_Dip,
        Pinky_Finger_Tip,
        HandLandmarkName_Max
    }

    private int[,] righthaand_CylinderIndexList = new int[21,2]
    {
        {(int)RightHandLandmarkName.Thumb_Mcp, (int)RightHandLandmarkName.Thumb_Cmc},
        {(int)RightHandLandmarkName.Thumb_Ip, (int)RightHandLandmarkName.Thumb_Mcp},
        {(int)RightHandLandmarkName.Thumb_Tip, (int)RightHandLandmarkName.Thumb_Ip},
        {(int)RightHandLandmarkName.Index_Finger_Pip, (int)RightHandLandmarkName.Index_Finger_Mcp},
        {(int)RightHandLandmarkName.Index_Finger_Dip, (int)RightHandLandmarkName.Index_Finger_Pip},
        {(int)RightHandLandmarkName.Index_Finger_Tip, (int)RightHandLandmarkName.Index_Finger_Dip},
        {(int)RightHandLandmarkName.Middle_Finger_Pip, (int)RightHandLandmarkName.Middle_Finger_Mcp},
        {(int)RightHandLandmarkName.Middle_Finger_Dip, (int)RightHandLandmarkName.Middle_Finger_Pip},
        {(int)RightHandLandmarkName.Middle_Finger_Tip, (int)RightHandLandmarkName.Middle_Finger_Dip},
        {(int)RightHandLandmarkName.Ring_Finger_Pip, (int)RightHandLandmarkName.Ring_Finger_Mcp},
        {(int)RightHandLandmarkName.Ring_Finger_Dip, (int)RightHandLandmarkName.Ring_Finger_Pip},
        {(int)RightHandLandmarkName.Ring_Finger_Tip, (int)RightHandLandmarkName.Ring_Finger_Dip},
        {(int)RightHandLandmarkName.Pinky_Finger_Pip, (int)RightHandLandmarkName.Pinky_Finger_Mcp},
        {(int)RightHandLandmarkName.Pinky_Finger_Dip, (int)RightHandLandmarkName.Pinky_Finger_Pip},
        {(int)RightHandLandmarkName.Pinky_Finger_Tip, (int)RightHandLandmarkName.Pinky_Finger_Dip},
        {(int)RightHandLandmarkName.Middle_Finger_Mcp, (int)RightHandLandmarkName.Index_Finger_Mcp},
        {(int)RightHandLandmarkName.Ring_Finger_Mcp, (int)RightHandLandmarkName.Middle_Finger_Mcp},
        {(int)RightHandLandmarkName.Pinky_Finger_Mcp, (int)RightHandLandmarkName.Ring_Finger_Mcp},
        {(int)RightHandLandmarkName.Thumb_Cmc, (int)RightHandLandmarkName.Wrist},
        {(int)RightHandLandmarkName.Index_Finger_Mcp, (int)RightHandLandmarkName.Wrist},
        {(int)RightHandLandmarkName.Pinky_Finger_Mcp, (int)RightHandLandmarkName.Wrist}
    };

    public static Queue<string> rcvData_righthand;
    public Queue<PartsData_righthand> moveData_righthand;
    [SerializeField] GameObject righthand_Wrist;
    [SerializeField] GameObject righthand_Thumb_Cmc;
    [SerializeField] GameObject righthand_Thumb_Mcp;
    [SerializeField] GameObject righthand_Thumb_Ip;
    [SerializeField] GameObject righthand_Thumb_Tip;
    [SerializeField] GameObject righthand_Index_Finger_Mcp;
    [SerializeField] GameObject righthand_Index_Finger_Pip;
    [SerializeField] GameObject righthand_Index_Finger_Dip;
    [SerializeField] GameObject righthand_Index_Finger_Tip;
    [SerializeField] GameObject righthand_Middle_Finger_Mcp;
    [SerializeField] GameObject righthand_Middle_Finger_Pip;
    [SerializeField] GameObject righthand_Middle_Finger_Dip;
    [SerializeField] GameObject righthand_Middle_Finger_Tip;
    [SerializeField] GameObject righthand_Ring_Finger_Mcp;
    [SerializeField] GameObject righthand_Ring_Finger_Pip;
    [SerializeField] GameObject righthand_Ring_Finger_Dip;
    [SerializeField] GameObject righthand_Ring_Finger_Tip;
    [SerializeField] GameObject righthand_Pinky_Finger_Mcp;
    [SerializeField] GameObject righthand_Pinky_Finger_Pip;
    [SerializeField] GameObject righthand_Pinky_Finger_Dip;
    [SerializeField] GameObject righthand_Pinky_Finger_Tip;

    private GameObject[] righthand_objs;

    private GameObject righthand_Cylinder_00;
    private GameObject righthand_Cylinder_01;
    private GameObject righthand_Cylinder_02;
    private GameObject righthand_Cylinder_03;
    private GameObject righthand_Cylinder_04;
    private GameObject righthand_Cylinder_05;
    private GameObject righthand_Cylinder_06;
    private GameObject righthand_Cylinder_07;
    private GameObject righthand_Cylinder_08;
    private GameObject righthand_Cylinder_09;
    private GameObject righthand_Cylinder_10;
    private GameObject righthand_Cylinder_11;
    private GameObject righthand_Cylinder_12;
    private GameObject righthand_Cylinder_13;
    private GameObject righthand_Cylinder_14;
    private GameObject righthand_Cylinder_15;
    private GameObject righthand_Cylinder_16;
    private GameObject righthand_Cylinder_17;
    private GameObject righthand_Cylinder_18;
    private GameObject righthand_Cylinder_19;
    private GameObject righthand_Cylinder_20;

    private GameObject[] righthand_Cylinder_objs;

    private int hidecnt;

    PartsData_righthand prev_moveData_righthand;
    private int rcvdata_devide; //受信データを分割して滑らかにする

    void Start()
    {
        hidecnt = 0;
        rcvdata_devide = 2;
        rcvData_righthand = new Queue<string>();
        moveData_righthand = new Queue<PartsData_righthand>();
        prev_moveData_righthand = null;

        righthand_objs = new GameObject[(int)RightHandLandmarkName.HandLandmarkName_Max];
        righthand_objs[(int)RightHandLandmarkName.Wrist] = righthand_Wrist;
        righthand_objs[(int)RightHandLandmarkName.Thumb_Cmc] = righthand_Thumb_Cmc;
        righthand_objs[(int)RightHandLandmarkName.Thumb_Mcp] = righthand_Thumb_Mcp;
        righthand_objs[(int)RightHandLandmarkName.Thumb_Ip] = righthand_Thumb_Ip;
        righthand_objs[(int)RightHandLandmarkName.Thumb_Tip] = righthand_Thumb_Tip;
        righthand_objs[(int)RightHandLandmarkName.Index_Finger_Mcp] = righthand_Index_Finger_Mcp;
        righthand_objs[(int)RightHandLandmarkName.Index_Finger_Pip] = righthand_Index_Finger_Pip;
        righthand_objs[(int)RightHandLandmarkName.Index_Finger_Dip] = righthand_Index_Finger_Dip;
        righthand_objs[(int)RightHandLandmarkName.Index_Finger_Tip] = righthand_Index_Finger_Tip;
        righthand_objs[(int)RightHandLandmarkName.Middle_Finger_Mcp] = righthand_Middle_Finger_Mcp;
        righthand_objs[(int)RightHandLandmarkName.Middle_Finger_Pip] = righthand_Middle_Finger_Pip;
        righthand_objs[(int)RightHandLandmarkName.Middle_Finger_Dip] = righthand_Middle_Finger_Dip;
        righthand_objs[(int)RightHandLandmarkName.Middle_Finger_Tip] = righthand_Middle_Finger_Tip;
        righthand_objs[(int)RightHandLandmarkName.Ring_Finger_Mcp] = righthand_Ring_Finger_Mcp;
        righthand_objs[(int)RightHandLandmarkName.Ring_Finger_Pip] = righthand_Ring_Finger_Pip;
        righthand_objs[(int)RightHandLandmarkName.Ring_Finger_Dip] = righthand_Ring_Finger_Dip;
        righthand_objs[(int)RightHandLandmarkName.Ring_Finger_Tip] = righthand_Ring_Finger_Tip;
        righthand_objs[(int)RightHandLandmarkName.Pinky_Finger_Mcp] = righthand_Pinky_Finger_Mcp;
        righthand_objs[(int)RightHandLandmarkName.Pinky_Finger_Pip] = righthand_Pinky_Finger_Pip;
        righthand_objs[(int)RightHandLandmarkName.Pinky_Finger_Dip] = righthand_Pinky_Finger_Dip;
        righthand_objs[(int)RightHandLandmarkName.Pinky_Finger_Tip] = righthand_Pinky_Finger_Tip;

        righthand_Cylinder_objs = new GameObject[righthaand_CylinderIndexList.GetLength(0)];
        righthand_Cylinder_objs[0] = righthand_Cylinder_00;
        righthand_Cylinder_objs[1] = righthand_Cylinder_01;
        righthand_Cylinder_objs[2] = righthand_Cylinder_02;
        righthand_Cylinder_objs[3] = righthand_Cylinder_03;
        righthand_Cylinder_objs[4] = righthand_Cylinder_04;
        righthand_Cylinder_objs[5] = righthand_Cylinder_05;
        righthand_Cylinder_objs[6] = righthand_Cylinder_06;
        righthand_Cylinder_objs[7] = righthand_Cylinder_07;
        righthand_Cylinder_objs[8] = righthand_Cylinder_08;
        righthand_Cylinder_objs[9] = righthand_Cylinder_09;
        righthand_Cylinder_objs[10] = righthand_Cylinder_10;
        righthand_Cylinder_objs[11] = righthand_Cylinder_11;
        righthand_Cylinder_objs[12] = righthand_Cylinder_12;
        righthand_Cylinder_objs[13] = righthand_Cylinder_13;
        righthand_Cylinder_objs[14] = righthand_Cylinder_14;
        righthand_Cylinder_objs[15] = righthand_Cylinder_15;
        righthand_Cylinder_objs[16] = righthand_Cylinder_16;
        righthand_Cylinder_objs[17] = righthand_Cylinder_17;
        righthand_Cylinder_objs[18] = righthand_Cylinder_18;
        righthand_Cylinder_objs[19] = righthand_Cylinder_19;
        righthand_Cylinder_objs[20] = righthand_Cylinder_20;

        for(int idx=0; idx<righthaand_CylinderIndexList.GetLength(0); idx++)
        {
            righthand_Cylinder_objs[idx] = GameObject.CreatePrimitive(PrimitiveType.Sphere);
            righthand_Cylinder_objs[idx].transform.localScale = new Vector3(0.1f, 0.1f, 0.1f);
        }
        ChangeVisibility(false);
    }

    void Update()
    {
        hidecnt++;
        if((rcvData_righthand != null) && (rcvData_righthand.Count != 0))
        {
            string rcvdata = rcvData_righthand.Dequeue();
            PartsData_righthand rcv_righthand = JsonUtility.FromJson<PartsData_righthand>(rcvdata);

            //今回値を前回値の差を、今回値(目標)に向かって数回に分けて移動させる
            if(prev_moveData_righthand != null)
            {
                PartsData_righthand div_righthand = new PartsData_righthand();

                //前回値を開始地点とする
                div_righthand.parts = prev_moveData_righthand.parts;
                for(int idx=(int)RightHandLandmarkName.Wrist; idx<(int)RightHandLandmarkName.HandLandmarkName_Max; idx++)
                {
                    div_righthand.x[idx] = prev_moveData_righthand.x[idx];
                    div_righthand.y[idx] = prev_moveData_righthand.y[idx];
                    div_righthand.z[idx] = prev_moveData_righthand.z[idx];
                }

                //今回値と前回値の差分を分割し、キューへ格納する
                for(int div=0; div<rcvdata_devide-1; div++)
                {
                    for(int idx=(int)RightHandLandmarkName.Wrist; idx<(int)RightHandLandmarkName.HandLandmarkName_Max; idx++)
                    {
                        div_righthand.x[idx] += (rcv_righthand.x[idx] - prev_moveData_righthand.x[idx]) / rcvdata_devide;
                        div_righthand.y[idx] += (rcv_righthand.y[idx] - prev_moveData_righthand.y[idx]) / rcvdata_devide;
                        div_righthand.z[idx] += (rcv_righthand.z[idx] - prev_moveData_righthand.z[idx]) / rcvdata_devide;
                    }
                    moveData_righthand.Enqueue(div_righthand);
                }
            }
            else
            {
                //ここを実行するのは初回のみ
                prev_moveData_righthand = new PartsData_righthand();
            }
            //分割数によっては最終値が中途半端になるため、最後は今回値ぴったりで終わらせる
            moveData_righthand.Enqueue(rcv_righthand);

            //今回値を前回値として保持
            prev_moveData_righthand.parts = rcv_righthand.parts;
            for(int idx=(int)RightHandLandmarkName.Wrist; idx<(int)RightHandLandmarkName.HandLandmarkName_Max; idx++)
            {
                prev_moveData_righthand.x[idx] = rcv_righthand.x[idx];
                prev_moveData_righthand.y[idx] = rcv_righthand.y[idx];
                prev_moveData_righthand.z[idx] = rcv_righthand.z[idx];
            }
        }

        if((moveData_righthand != null) && (moveData_righthand.Count != 0))
        {
            PartsData_righthand right_hand = moveData_righthand.Dequeue();
            hidecnt = 0;

            //右手に連動する
            for(int idx=(int)RightHandLandmarkName.Wrist; idx<(int)RightHandLandmarkName.HandLandmarkName_Max; idx++)
            {
                Transform myTransform;
                myTransform = righthand_objs[idx].transform;
                Vector3 pos = myTransform.position;
                pos.x = right_hand.x[idx]*(-10f)+5f;
                pos.y = right_hand.y[idx]*(-6f)+3f;
                pos.z = right_hand.z[idx]*10f-3;
                myTransform.position = pos;
            }
            LandmarkJoint(righthand_objs, righthand_Cylinder_objs);
            ChangeVisibility(true);
        }
        if(hidecnt > 100)
        {
            hidecnt = 0;
            ChangeVisibility(false);
            rcvData_righthand.Clear();
            moveData_righthand.Clear();
        }
    }

    private void OnApplicationQuit()
    {
        rcvData_righthand.Clear();
        moveData_righthand.Clear();
    }

    private void OnDestroy()
    {
        rcvData_righthand.Clear();
        moveData_righthand.Clear();
    }

    public static void SetRcvData(string rcvdata)
    {
        rcvData_righthand.Enqueue(rcvdata);
    }

    private void LandmarkJoint(GameObject[] hand_objs, GameObject[] cylinder_objs)
    {
        for(int idx=0; idx<righthaand_CylinderIndexList.GetLength(0); idx++)
        {
            cylinder_objs[idx].transform.position = Vector3.Lerp(hand_objs[righthaand_CylinderIndexList[idx,0]].transform.position, hand_objs[righthaand_CylinderIndexList[idx,1]].transform.position, 0.5f);
            cylinder_objs[idx].transform.LookAt(hand_objs[righthaand_CylinderIndexList[idx,1]].transform.position);
            cylinder_objs[idx].transform.localScale = new Vector3(0.2f, 0.2f, (hand_objs[righthaand_CylinderIndexList[idx,0]].transform.position - hand_objs[righthaand_CylinderIndexList[idx,1]].transform.position).magnitude);
        }
    }

    private void ChangeVisibility(bool vis)
    {
        for(int idx=(int)RightHandLandmarkName.Wrist; idx<(int)RightHandLandmarkName.HandLandmarkName_Max; idx++)
        {
            righthand_objs[idx].GetComponent<Renderer>().enabled = vis;
        }
        for(int idx=0; idx<righthaand_CylinderIndexList.GetLength(0); idx++)
        {
            righthand_Cylinder_objs[idx].GetComponent<Renderer>().enabled = vis;
        }
    }
}
}

変更点を見ていきます。

    private int[,] righthaand_CylinderIndexList = new int[21,2]
    {
        {(int)RightHandLandmarkName.Thumb_Mcp, (int)RightHandLandmarkName.Thumb_Cmc},
        {(int)RightHandLandmarkName.Thumb_Ip, (int)RightHandLandmarkName.Thumb_Mcp},
        {(int)RightHandLandmarkName.Thumb_Tip, (int)RightHandLandmarkName.Thumb_Ip},
        {(int)RightHandLandmarkName.Index_Finger_Pip, (int)RightHandLandmarkName.Index_Finger_Mcp},
        {(int)RightHandLandmarkName.Index_Finger_Dip, (int)RightHandLandmarkName.Index_Finger_Pip},
        {(int)RightHandLandmarkName.Index_Finger_Tip, (int)RightHandLandmarkName.Index_Finger_Dip},
        {(int)RightHandLandmarkName.Middle_Finger_Pip, (int)RightHandLandmarkName.Middle_Finger_Mcp},
        {(int)RightHandLandmarkName.Middle_Finger_Dip, (int)RightHandLandmarkName.Middle_Finger_Pip},
        {(int)RightHandLandmarkName.Middle_Finger_Tip, (int)RightHandLandmarkName.Middle_Finger_Dip},
        {(int)RightHandLandmarkName.Ring_Finger_Pip, (int)RightHandLandmarkName.Ring_Finger_Mcp},
        {(int)RightHandLandmarkName.Ring_Finger_Dip, (int)RightHandLandmarkName.Ring_Finger_Pip},
        {(int)RightHandLandmarkName.Ring_Finger_Tip, (int)RightHandLandmarkName.Ring_Finger_Dip},
        {(int)RightHandLandmarkName.Pinky_Finger_Pip, (int)RightHandLandmarkName.Pinky_Finger_Mcp},
        {(int)RightHandLandmarkName.Pinky_Finger_Dip, (int)RightHandLandmarkName.Pinky_Finger_Pip},
        {(int)RightHandLandmarkName.Pinky_Finger_Tip, (int)RightHandLandmarkName.Pinky_Finger_Dip},
        {(int)RightHandLandmarkName.Middle_Finger_Mcp, (int)RightHandLandmarkName.Index_Finger_Mcp},
        {(int)RightHandLandmarkName.Ring_Finger_Mcp, (int)RightHandLandmarkName.Middle_Finger_Mcp},
        {(int)RightHandLandmarkName.Pinky_Finger_Mcp, (int)RightHandLandmarkName.Ring_Finger_Mcp},
        {(int)RightHandLandmarkName.Thumb_Cmc, (int)RightHandLandmarkName.Wrist},
        {(int)RightHandLandmarkName.Index_Finger_Mcp, (int)RightHandLandmarkName.Wrist},
        {(int)RightHandLandmarkName.Pinky_Finger_Mcp, (int)RightHandLandmarkName.Wrist}
    };

どこのLandmarkどうしを接続するかを定義するindexリストです。

    void Start()
    {
・・・
        for(int idx=0; idx<righthaand_CylinderIndexList.GetLength(0); idx++)
        {
            righthand_Cylinder_objs[idx] = GameObject.CreatePrimitive(PrimitiveType.Sphere);
            righthand_Cylinder_objs[idx].transform.localScale = new Vector3(0.1f, 0.1f, 0.1f);
        }

Landmarkどうしの接続に使用するオブジェクトは、ここで動的に生成しています。
オブジェクトをヒエラルキーに用意しておくか、動的に生成するか、どちらでもOKです。

    private void LandmarkJoint(GameObject[] hand_objs, GameObject[] cylinder_objs)
    {
        for(int idx=0; idx<righthaand_CylinderIndexList.GetLength(0); idx++)
        {
            cylinder_objs[idx].transform.position = Vector3.Lerp(hand_objs[righthaand_CylinderIndexList[idx,0]].transform.position, hand_objs[righthaand_CylinderIndexList[idx,1]].transform.position, 0.5f);
            cylinder_objs[idx].transform.LookAt(hand_objs[righthaand_CylinderIndexList[idx,1]].transform.position);
            cylinder_objs[idx].transform.localScale = new Vector3(0.2f, 0.2f, (hand_objs[righthaand_CylinderIndexList[idx,0]].transform.position - hand_objs[righthaand_CylinderIndexList[idx,1]].transform.position).magnitude);
        }
    }

2点のLandmarkをオブジェクトで接続します。
2点の座標の中間点は、Vector3.Lerp()で求められます。
LookAtで、オブジェクトの正面(ローカルZ軸プラス方向)をLandmarkの方向に向けます。
localScaleでZ軸を可変します。長さは (座標1 – 座標2).magnitude で求められます。

左手パーツ管理

using System.Collections.Generic;
using UnityEngine;

namespace PartsData_lefthand_ns{

[System.Serializable]
public class PartsData_lefthand
{
    public string parts = "";
    public float[] x= new float[21];
    public float[] y= new float[21];
    public float[] z= new float[21];
}

public class LeftHandManager : MonoBehaviour
{
    public enum LeftHandLandmarkName
    {
        Wrist = 0,
        Thumb_Cmc,
        Thumb_Mcp,
        Thumb_Ip,
        Thumb_Tip,
        Index_Finger_Mcp,
        Index_Finger_Pip,
        Index_Finger_Dip,
        Index_Finger_Tip,
        Middle_Finger_Mcp,
        Middle_Finger_Pip,
        Middle_Finger_Dip,
        Middle_Finger_Tip,
        Ring_Finger_Mcp,
        Ring_Finger_Pip,
        Ring_Finger_Dip,
        Ring_Finger_Tip,
        Pinky_Finger_Mcp,
        Pinky_Finger_Pip,
        Pinky_Finger_Dip,
        Pinky_Finger_Tip,
        HandLandmarkName_Max
    }

    private int[,] lefthaand_CylinderIndexList = new int[21,2]
    {
        {(int)LeftHandLandmarkName.Thumb_Mcp, (int)LeftHandLandmarkName.Thumb_Cmc},
        {(int)LeftHandLandmarkName.Thumb_Ip, (int)LeftHandLandmarkName.Thumb_Mcp},
        {(int)LeftHandLandmarkName.Thumb_Tip, (int)LeftHandLandmarkName.Thumb_Ip},
        {(int)LeftHandLandmarkName.Index_Finger_Pip, (int)LeftHandLandmarkName.Index_Finger_Mcp},
        {(int)LeftHandLandmarkName.Index_Finger_Dip, (int)LeftHandLandmarkName.Index_Finger_Pip},
        {(int)LeftHandLandmarkName.Index_Finger_Tip, (int)LeftHandLandmarkName.Index_Finger_Dip},
        {(int)LeftHandLandmarkName.Middle_Finger_Pip, (int)LeftHandLandmarkName.Middle_Finger_Mcp},
        {(int)LeftHandLandmarkName.Middle_Finger_Dip, (int)LeftHandLandmarkName.Middle_Finger_Pip},
        {(int)LeftHandLandmarkName.Middle_Finger_Tip, (int)LeftHandLandmarkName.Middle_Finger_Dip},
        {(int)LeftHandLandmarkName.Ring_Finger_Pip, (int)LeftHandLandmarkName.Ring_Finger_Mcp},
        {(int)LeftHandLandmarkName.Ring_Finger_Dip, (int)LeftHandLandmarkName.Ring_Finger_Pip},
        {(int)LeftHandLandmarkName.Ring_Finger_Tip, (int)LeftHandLandmarkName.Ring_Finger_Dip},
        {(int)LeftHandLandmarkName.Pinky_Finger_Pip, (int)LeftHandLandmarkName.Pinky_Finger_Mcp},
        {(int)LeftHandLandmarkName.Pinky_Finger_Dip, (int)LeftHandLandmarkName.Pinky_Finger_Pip},
        {(int)LeftHandLandmarkName.Pinky_Finger_Tip, (int)LeftHandLandmarkName.Pinky_Finger_Dip},
        {(int)LeftHandLandmarkName.Middle_Finger_Mcp, (int)LeftHandLandmarkName.Index_Finger_Mcp},
        {(int)LeftHandLandmarkName.Ring_Finger_Mcp, (int)LeftHandLandmarkName.Middle_Finger_Mcp},
        {(int)LeftHandLandmarkName.Pinky_Finger_Mcp, (int)LeftHandLandmarkName.Ring_Finger_Mcp},
        {(int)LeftHandLandmarkName.Thumb_Cmc, (int)LeftHandLandmarkName.Wrist},
        {(int)LeftHandLandmarkName.Index_Finger_Mcp, (int)LeftHandLandmarkName.Wrist},
        {(int)LeftHandLandmarkName.Pinky_Finger_Mcp, (int)LeftHandLandmarkName.Wrist}
    };

    public static Queue<string> rcvData_lefthand;
    public Queue<PartsData_lefthand> moveData_lefthand;
    [SerializeField] GameObject lefthand_Wrist;
    [SerializeField] GameObject lefthand_Thumb_Cmc;
    [SerializeField] GameObject lefthand_Thumb_Mcp;
    [SerializeField] GameObject lefthand_Thumb_Ip;
    [SerializeField] GameObject lefthand_Thumb_Tip;
    [SerializeField] GameObject lefthand_Index_Finger_Mcp;
    [SerializeField] GameObject lefthand_Index_Finger_Pip;
    [SerializeField] GameObject lefthand_Index_Finger_Dip;
    [SerializeField] GameObject lefthand_Index_Finger_Tip;
    [SerializeField] GameObject lefthand_Middle_Finger_Mcp;
    [SerializeField] GameObject lefthand_Middle_Finger_Pip;
    [SerializeField] GameObject lefthand_Middle_Finger_Dip;
    [SerializeField] GameObject lefthand_Middle_Finger_Tip;
    [SerializeField] GameObject lefthand_Ring_Finger_Mcp;
    [SerializeField] GameObject lefthand_Ring_Finger_Pip;
    [SerializeField] GameObject lefthand_Ring_Finger_Dip;
    [SerializeField] GameObject lefthand_Ring_Finger_Tip;
    [SerializeField] GameObject lefthand_Pinky_Finger_Mcp;
    [SerializeField] GameObject lefthand_Pinky_Finger_Pip;
    [SerializeField] GameObject lefthand_Pinky_Finger_Dip;
    [SerializeField] GameObject lefthand_Pinky_Finger_Tip;

    GameObject[] lefthand_objs;

    private GameObject lefthand_Cylinder_00;
    private GameObject lefthand_Cylinder_01;
    private GameObject lefthand_Cylinder_02;
    private GameObject lefthand_Cylinder_03;
    private GameObject lefthand_Cylinder_04;
    private GameObject lefthand_Cylinder_05;
    private GameObject lefthand_Cylinder_06;
    private GameObject lefthand_Cylinder_07;
    private GameObject lefthand_Cylinder_08;
    private GameObject lefthand_Cylinder_09;
    private GameObject lefthand_Cylinder_10;
    private GameObject lefthand_Cylinder_11;
    private GameObject lefthand_Cylinder_12;
    private GameObject lefthand_Cylinder_13;
    private GameObject lefthand_Cylinder_14;
    private GameObject lefthand_Cylinder_15;
    private GameObject lefthand_Cylinder_16;
    private GameObject lefthand_Cylinder_17;
    private GameObject lefthand_Cylinder_18;
    private GameObject lefthand_Cylinder_19;
    private GameObject lefthand_Cylinder_20;

    private GameObject[] lefthand_Cylinder_objs;

    private int hidecnt;

    PartsData_lefthand prev_moveData_lefthand;
    private int rcvdata_devide; //受信データを分割して滑らかにする

    void Start()
    {
        hidecnt = 0;
        rcvdata_devide = 2;
        rcvData_lefthand = new Queue<string>();
        moveData_lefthand = new Queue<PartsData_lefthand>();
        prev_moveData_lefthand = null;

        lefthand_objs = new GameObject[(int)LeftHandLandmarkName.HandLandmarkName_Max];
        lefthand_objs[(int)LeftHandLandmarkName.Wrist] = lefthand_Wrist;
        lefthand_objs[(int)LeftHandLandmarkName.Thumb_Cmc] = lefthand_Thumb_Cmc;
        lefthand_objs[(int)LeftHandLandmarkName.Thumb_Mcp] = lefthand_Thumb_Mcp;
        lefthand_objs[(int)LeftHandLandmarkName.Thumb_Ip] = lefthand_Thumb_Ip;
        lefthand_objs[(int)LeftHandLandmarkName.Thumb_Tip] = lefthand_Thumb_Tip;
        lefthand_objs[(int)LeftHandLandmarkName.Index_Finger_Mcp] = lefthand_Index_Finger_Mcp;
        lefthand_objs[(int)LeftHandLandmarkName.Index_Finger_Pip] = lefthand_Index_Finger_Pip;
        lefthand_objs[(int)LeftHandLandmarkName.Index_Finger_Dip] = lefthand_Index_Finger_Dip;
        lefthand_objs[(int)LeftHandLandmarkName.Index_Finger_Tip] = lefthand_Index_Finger_Tip;
        lefthand_objs[(int)LeftHandLandmarkName.Middle_Finger_Mcp] = lefthand_Middle_Finger_Mcp;
        lefthand_objs[(int)LeftHandLandmarkName.Middle_Finger_Pip] = lefthand_Middle_Finger_Pip;
        lefthand_objs[(int)LeftHandLandmarkName.Middle_Finger_Dip] = lefthand_Middle_Finger_Dip;
        lefthand_objs[(int)LeftHandLandmarkName.Middle_Finger_Tip] = lefthand_Middle_Finger_Tip;
        lefthand_objs[(int)LeftHandLandmarkName.Ring_Finger_Mcp] = lefthand_Ring_Finger_Mcp;
        lefthand_objs[(int)LeftHandLandmarkName.Ring_Finger_Pip] = lefthand_Ring_Finger_Pip;
        lefthand_objs[(int)LeftHandLandmarkName.Ring_Finger_Dip] = lefthand_Ring_Finger_Dip;
        lefthand_objs[(int)LeftHandLandmarkName.Ring_Finger_Tip] = lefthand_Ring_Finger_Tip;
        lefthand_objs[(int)LeftHandLandmarkName.Pinky_Finger_Mcp] = lefthand_Pinky_Finger_Mcp;
        lefthand_objs[(int)LeftHandLandmarkName.Pinky_Finger_Pip] = lefthand_Pinky_Finger_Pip;
        lefthand_objs[(int)LeftHandLandmarkName.Pinky_Finger_Dip] = lefthand_Pinky_Finger_Dip;
        lefthand_objs[(int)LeftHandLandmarkName.Pinky_Finger_Tip] = lefthand_Pinky_Finger_Tip;

        lefthand_Cylinder_objs = new GameObject[lefthaand_CylinderIndexList.GetLength(0)];
        lefthand_Cylinder_objs[0] = lefthand_Cylinder_00;
        lefthand_Cylinder_objs[1] = lefthand_Cylinder_01;
        lefthand_Cylinder_objs[2] = lefthand_Cylinder_02;
        lefthand_Cylinder_objs[3] = lefthand_Cylinder_03;
        lefthand_Cylinder_objs[4] = lefthand_Cylinder_04;
        lefthand_Cylinder_objs[5] = lefthand_Cylinder_05;
        lefthand_Cylinder_objs[6] = lefthand_Cylinder_06;
        lefthand_Cylinder_objs[7] = lefthand_Cylinder_07;
        lefthand_Cylinder_objs[8] = lefthand_Cylinder_08;
        lefthand_Cylinder_objs[9] = lefthand_Cylinder_09;
        lefthand_Cylinder_objs[10] = lefthand_Cylinder_10;
        lefthand_Cylinder_objs[11] = lefthand_Cylinder_11;
        lefthand_Cylinder_objs[12] = lefthand_Cylinder_12;
        lefthand_Cylinder_objs[13] = lefthand_Cylinder_13;
        lefthand_Cylinder_objs[14] = lefthand_Cylinder_14;
        lefthand_Cylinder_objs[15] = lefthand_Cylinder_15;
        lefthand_Cylinder_objs[16] = lefthand_Cylinder_16;
        lefthand_Cylinder_objs[17] = lefthand_Cylinder_17;
        lefthand_Cylinder_objs[18] = lefthand_Cylinder_18;
        lefthand_Cylinder_objs[19] = lefthand_Cylinder_19;
        lefthand_Cylinder_objs[20] = lefthand_Cylinder_20;

        for(int idx=0; idx<lefthaand_CylinderIndexList.GetLength(0); idx++)
        {
            lefthand_Cylinder_objs[idx] = GameObject.CreatePrimitive(PrimitiveType.Sphere);
            lefthand_Cylinder_objs[idx].transform.localScale = new Vector3(0.1f, 0.1f, 0.1f);
        }
        ChangeVisibility(false);
    }

    void Update()
    {
        hidecnt++;
        if((rcvData_lefthand != null) && (rcvData_lefthand.Count != 0))
        {
            string rcvdata = rcvData_lefthand.Dequeue();
            PartsData_lefthand rcv_lefthand = JsonUtility.FromJson<PartsData_lefthand>(rcvdata);

            //今回値を前回値の差を、今回値(目標)に向かって数回に分けて移動させる
            if(prev_moveData_lefthand != null)
            {
                PartsData_lefthand div_lefthand = new PartsData_lefthand();

                //前回値を開始地点とする
                div_lefthand.parts = prev_moveData_lefthand.parts;
                for(int idx=(int)LeftHandLandmarkName.Wrist; idx<(int)LeftHandLandmarkName.HandLandmarkName_Max; idx++)
                {
                    div_lefthand.x[idx] = prev_moveData_lefthand.x[idx];
                    div_lefthand.y[idx] = prev_moveData_lefthand.y[idx];
                    div_lefthand.z[idx] = prev_moveData_lefthand.z[idx];
                }

                //今回値と前回値の差分を分割し、キューへ格納する
                for(int div=0; div<rcvdata_devide-1; div++)
                {
                    for(int idx=(int)LeftHandLandmarkName.Wrist; idx<(int)LeftHandLandmarkName.HandLandmarkName_Max; idx++)
                    {
                        div_lefthand.x[idx] += (rcv_lefthand.x[idx] - prev_moveData_lefthand.x[idx]) / rcvdata_devide;
                        div_lefthand.y[idx] += (rcv_lefthand.y[idx] - prev_moveData_lefthand.y[idx]) / rcvdata_devide;
                        div_lefthand.z[idx] += (rcv_lefthand.z[idx] - prev_moveData_lefthand.z[idx]) / rcvdata_devide;
                    }
                    moveData_lefthand.Enqueue(div_lefthand);
                }
            }
            else
            {
                //ここを実行するのは初回のみ
                prev_moveData_lefthand = new PartsData_lefthand();
            }
            //分割数によっては最終値が中途半端になるため、最後は今回値ぴったりで終わらせる
            moveData_lefthand.Enqueue(rcv_lefthand);
            
            //今回値を前回値として保持
            prev_moveData_lefthand.parts = rcv_lefthand.parts;
            for(int idx=(int)LeftHandLandmarkName.Wrist; idx<(int)LeftHandLandmarkName.HandLandmarkName_Max; idx++)
            {
                prev_moveData_lefthand.x[idx] = rcv_lefthand.x[idx];
                prev_moveData_lefthand.y[idx] = rcv_lefthand.y[idx];
                prev_moveData_lefthand.z[idx] = rcv_lefthand.z[idx];
            }
        }

        if((moveData_lefthand != null) && (moveData_lefthand.Count != 0))
        {
            PartsData_lefthand left_hand = moveData_lefthand.Dequeue();
            hidecnt = 0;

            //左手に連動する
            for(int idx=(int)LeftHandLandmarkName.Wrist; idx<(int)LeftHandLandmarkName.HandLandmarkName_Max; idx++)
            {
                Transform myTransform;
                myTransform = lefthand_objs[idx].transform;
                Vector3 pos = myTransform.position;
                pos.x = left_hand.x[idx]*(-10f)+5f;
                pos.y = left_hand.y[idx]*(-6f)+3f;
                pos.z = left_hand.z[idx]*10f-3;
                myTransform.position = pos;
            }
            LandmarkJoint(lefthand_objs, lefthand_Cylinder_objs);
            ChangeVisibility(true);
        }
        if(hidecnt > 100)
        {
            hidecnt = 0;
            ChangeVisibility(false);
            rcvData_lefthand.Clear();
            moveData_lefthand.Clear();
        }
    }

    private void OnApplicationQuit()
    {
        rcvData_lefthand.Clear();
        moveData_lefthand.Clear();
    }

    private void OnDestroy()
    {
        rcvData_lefthand.Clear();
        moveData_lefthand.Clear();
    }

    public static void SetRcvData(string rcvdata)
    {
        rcvData_lefthand.Enqueue(rcvdata);
    }

    private void LandmarkJoint(GameObject[] hand_objs, GameObject[] cylinder_objs)
    {
        for(int idx=0; idx<lefthaand_CylinderIndexList.GetLength(0); idx++)
        {
            cylinder_objs[idx].transform.position = Vector3.Lerp(hand_objs[lefthaand_CylinderIndexList[idx,0]].transform.position, hand_objs[lefthaand_CylinderIndexList[idx,1]].transform.position, 0.5f);
            cylinder_objs[idx].transform.LookAt(hand_objs[lefthaand_CylinderIndexList[idx,1]].transform.position);
            cylinder_objs[idx].transform.localScale = new Vector3(0.2f, 0.2f, (hand_objs[lefthaand_CylinderIndexList[idx,0]].transform.position - hand_objs[lefthaand_CylinderIndexList[idx,1]].transform.position).magnitude);
        }
    }

    private void ChangeVisibility(bool vis)
    {
        for(int idx=(int)LeftHandLandmarkName.Wrist; idx<(int)LeftHandLandmarkName.HandLandmarkName_Max; idx++)
        {
            lefthand_objs[idx].GetComponent<Renderer>().enabled = vis;
        }
        for(int idx=0; idx<lefthaand_CylinderIndexList.GetLength(0); idx++)
        {
            lefthand_Cylinder_objs[idx].GetComponent<Renderer>().enabled = vis;
        }
    }
}
}

変更点は右手と同様です。


エディタ設定

BlendShapeが設定されているオブジェクトを選択すると、インスペクターにスライダーが表示されます。これを動かすことで、モデルがどのように動くかを確認できます。
値は0〜100の範囲になっています。
インスペクターのBlendShapeに並んでいる上から順に、0から始まる数字(ID)が設定されています。
これをスクリプトのSetBlendShapeWeight()の第1引数に指定します。

動作確認

眉・瞬き・口をBlendShapeで動かすことができました。
目と口はもう少し開くようにしても良いかも🤔
手も輪郭が付いたことで、少し人間に近づきましたね。

このアバターは味はあるかもしれませんが、使い続けるのはキツいので、もう少しちゃんと作り直したいです😭

BlenderとUnityの回転軸

ここ数日、Blenderのモデルを使う上で、回転軸の問題に詰まっていました。
Blenderの座標系は左手系


Unityは右手系


この違いは、Unityへインポートしたときに自動的に変換されるのですが、それが原因で意図しない向きになってしまうことがあります。
Unityにインポートしたときの回転を確認して、Blenderでそれを補正するようにモデルを回転させてエクスポートすると良いですね。

さいごに

アバターの顔を、MediaPipeのBlendShapeで動かすことができました。
手のモデルは、もう少し考えたら何とかなりそうな気がするので、頑張ってみます。
アバターの顔も何とかします😅

それでは、今回はここまで。
ありがとうございました😊

コメント

タイトルとURLをコピーしました