Difference between revisions of "Binary File Formats"

From Wolfire Games Wiki
Jump to: navigation, search
(Add section header information to PHXBN file layout tables)
(Add more info about how bones_matrices gets calculated)
Line 453: Line 453:
 
|-
 
|-
 
! bones_center_of_mass
 
! bones_center_of_mass
| 12 bytes * bones_count || float32[bones_count][3] || only present if version is 7 or higher || if version is less than 7 then calculate from bones_ends midpoints instead
+
| 12 bytes * bones_count || float32[bones_count][3] || only present if version is 7 or higher || if version is less than 7 then calculate from '''bones_ends''' midpoints instead
 
|-
 
|-
 
! bones_matrices
 
! bones_matrices
| 144 bytes * bones_count || float32[bones_count][16] || only present if version is 9 or higher || this value is ignored and recalculated internally in engine regardless of version
+
| 144 bytes * bones_count || float32[bones_count][16] || only present if version is 9 or higher || this value is ignored and recalculated internally in engine regardless of version<br>the engine uses '''bones_ends''' to calculate it
 
|}
 
|}
  
 
-----
 
-----
 +
 
====PHXBN control joints====
 
====PHXBN control joints====
  

Revision as of 00:18, 8 March 2021

These are (some of) the custom binary file formats that are used by Wolfire's games (the Phoenix engine for Overgrowth, and Lugaru).

They are in 010 editor template format (which is not a free product). This format is quite a lot like C code, though not exactly C.

NOTE: If there's an open source tool and/or binary file description format for doing similar things (besides just writing out C code), then please let Wolfire know!

TODO: If there are any proprietary binary file formats missing that you know of, please let Wolfire know!

LGSOLID

This is the model file format for models in the Lugaru engine.

//------------------------------------------------
//--- 010 Editor v7.0.2 Binary Template
//
//      File: LGSOLID.bt
//   Authors: Wolfire Games
//   Version: 1.0
//   Purpose: Model data for Lugaru engine
//  Category: 3D
// File Mask: *.solid
//   History: 
//   1.0   Initial release
//------------------------------------------------

struct {  // file
  BigEndian();
  short vertex_count;
  short triangle_count;

  struct {  // vertices
    float x;
    float y;
    float z;
  } vertices[vertex_count];

  struct {  // triangles
    short vertex_index;
    short _unused <hidden=true>;
    short vertex_index;
    short _unused <hidden=true>;
    short vertex_index;
    short _unused <hidden=true>;

    float gx[3];
    float gy[3];
  } triangles[triangle_count];
} file;

PHXANM

This is the animation file format for rigged objects in the Phoenix engine.

//------------------------------------------------
//--- 010 Editor v7.0.2 Binary Template
//
//      File: PHXANM.bt
//   Authors: Wolfire Games
//   Version: 1.0
//   Purpose: Animation for Phoenix Engine
//  Category: 3D
// File Mask: *.anm
//   History:
//   1.0   Initial release
//------------------------------------------------

struct vec3 {
    float x;
    float y;
    float z;
};

struct mat4x4 {
    float components[16];
};

struct { // file
    struct { // header
        int version;

        if (version >= 10) {
            char centered;
        } else {
            local char centered = false;
        }

        if (version >= 1) {
            char looping;
        } else {
            local char looping = true;
        }

        if (version > 0 && version < 11) {
            int start;
        } else {
            local int start = version;
        }

        int total_length;
    } header <bgcolor=cLtGray>;

    struct { // keyframes
        int frame_count;

        struct { // frames
            int time;

            if (header.version >= 4) {
                int num_weights;
                float weights[num_weights];
            }

            int num_bone_matrices;
            mat4x4 bone_matrices[num_bone_matrices];

            if (header.version >= 7) {
                int num_weapon_matrices;

                if (header.version >= 8) {
                    struct {
                        mat4x4 matrix;
                        int relative_id;
                        float relative_weight;
                    } weapon_matrices[num_weapon_matrices];
                } else {
                    mat4x4 weapon_matrices[num_weapon_matrices];
                }
            }

            if (header.version >= 9) {
                char use_mobility;

                if (use_mobility) {
                    mat4x4 mobility_matrix;
                }
            }

            if (header.version >= 2) {
                int num_events;

                struct {
                    int which_bone_index;
                    int event_name_size;
                    char event_name[event_name_size] <optimize=false>;
                } events[num_events];
            }

            if (header.version >= 3) {
                int num_ik_bones;

                struct {
                    vec3 ignored1;
                    vec3 ignored2;
                    int num_paths;
                    int paths[num_paths];
                    int ik_bone_label_size;
                    char ik_bone_label[ik_bone_label_size] <optimize=false>;
                } ik_bones[num_ik_bones] <optimize=false>;
            }

            if (header.version >= 5) {
                int num_shape_keys;

                struct {
                    float weight;
                    int shape_label_size;
                    char label[shape_label_size] <optimize=false>;
                } shape_keys[num_shape_keys] <optimize=false>;
            }

            if (header.version >= 6) {
                int num_status_keys;

                struct {
                    float weight;
                    int status_key_label_size;
                    char label[status_key_label_size] <optimize=false>;
                } status_keys[num_status_keys] <optimize=false>;
            }

            if (header.centered) {
                float rotation;
                float center_offset;
            }
        } frames[frame_count] <optimize=false>;
    } keyframes;
} file;

PHXBN

This is the skeleton file format for rigged objects in the Phoenix engine.

PHXBN 010 template

For use with the 010 hex editor program

//------------------------------------------------
//--- 010 Editor v7.0.2 Binary Template
//
//      File: PHXBN.bt
//   Authors: Wolfire Games
//   Version: 1.0
//   Purpose: Animation Bones for Phoenix Engine
//  Category: 3D
// File Mask: *.phxbn
//   History:
//   1.0   Initial release
//------------------------------------------------

struct vec3 {
    float x;
    float y;
    float z;
};

struct vec4 {
    float x;
    float y;
    float z;
    float w;
};

struct mat4x4 {
    float components[16];
};

enum RiggingStage {
    nothing = -1,
    create_bones = 0,
    control_joints = 1,
    animate = 2,
    pose_weights = 3,
};

enum JointType {
    hinge_joint = 0,
    amotor_joint = 1,
    fixed_joint = 2,
};

typedef struct {
    JointType joint_type;

    if (joint_type == amotor_joint) {
        float amotor_stop_angle[6];
    } else if (joint_type == hinge_joint) {
        float hinge_stop_angle[2];
    }

    int bone_id[2];

    if (joint_type == hinge_joint) {
        vec3 axis;
    }
} JOINTRECORD <size=SizeJOINT>;

int SizeJOINT(JOINTRECORD& j) {
    return sizeof(JointType) +
        sizeof(int) * 2 +  // bone_id
        (ReadInt(startof(j)) == amotor_joint ? sizeof(float) * 6 : 0) +
        (ReadInt(startof(j)) == hinge_joint ? (sizeof(float) * 2 + sizeof(vec3)) : 0);
}

typedef struct {
    int bone_id;
    int chain_length;
    int name_length;
    char name[name_length];
}  SIMPLE_IK_BONE_RECORD <size=SizeSIMPLE_IK_BONE_RECORD>;

int SizeSIMPLE_IK_BONE_RECORD(SIMPLE_IK_BONE_RECORD& b) {
    return sizeof(int) + // bone_id
        sizeof(int) +  // chain_length
        sizeof(int) +  // name_length
        (ReadInt(startof(b) + sizeof(int) * 2));  // name
}

struct { // file
    struct { // header
        int version;  // If version >= 6, this field exists
        RiggingStage rigging_stage;

        if (rigging_stage == animate) {
            rigging_stage = control_joints;
        }
    } header <bgcolor=cLtGray>;

    struct { // points
        int count;
        vec3 points[count];

        if (header.version >= 8) {
            int point_parent_indices[count];
        }
    } points <bgcolor=cLtGreen>;

    struct { // bones
        int count;

        struct BONE_END {
            int index_first;
            int index_second;
        } bone_ends[count];

        if (header.version >= 8) {
            int bone_parent_indices[count];
        }

        if (header.version >= 6) {
            float bone_masses[count];
            // Also loads shared mass data from a different file, if provided
        }

        if (header.version >= 7) {
            vec3 bone_com[count];
        } else {
            // bone_com = bone-ends midpoints, calculated from above data
            // no file space taken up here
        }

        if (header.version >= 9) {
            mat4x4 _bone_mat_ignored[count];  // ignored, calcuated elsewhere
        }
    } bones <bgcolor=cLtBlue>;

    struct { // control_joints
        if (header.rigging_stage == control_joints) {
            if (header.version >= 11) {
                // NOTE: There's no way to get this if version < 11. The game just grabbed it from the model instead
                int num_vertices;
            } else {
                // NOTE: If you're using this in template in 010 and opening a file with version < 11,
                // you have to edit this line, and hard code this to 3x the face count of the model
                local int num_vertices = 17502;
            }

            vec4 bone_weights[num_vertices];
            vec4 bone_ids[num_vertices];

            int hier_parents_bone_ids[bones.count];

            int num_joints;

            struct JOINTS {
                JOINTRECORD records[num_joints] <optimize=false>;
            } joints;
        }
    } control_joints <bgcolor=cLtAqua>;

    if (header.version >= 10) {
        struct { // simple_ik_bones
            int count;
            SIMPLE_IK_BONE_RECORD records[count] <optimize=false>;
        } simple_ik_bones;
    }
} file;

PHXBN file layout

This table format may be easier to understand for someone not familiar with C structs and 010's template file format. I'm familiar with both, and I still think it might be easier to read. -merlyn


PHXBN enumerated types

Some enumerated type definitions. These are not part of the file data itself, and are just assumed information:

RiggingStage enumerated type
Name Value Notes
nothing -1
create_bones 0
control_joints 1
animate 2 if this is present in the file, the engine ignores it and pretends the value is control_joints (1)
pose_weights 3
JointType enumerated type
Name Value Notes
hinge_joint 0
amotor_joint 1
fixed_joint 2

PHXBN header

The file data itself begins here:

skeleton main header
Name Size in Bytes Type of data When Present Notes
version 4 bytes int32 only present if version is 6 or higher if version is less than 6 then assume version is 5 and assign this value to rigging_stage instead
rigging_stage 4 bytes enum RiggingStage (int32) if version is less than 6 then this is the first 4 bytes of the file if value is 2 (animate) then it is ignored and reinterpreted as 1 (control_joints)
skeleton points
Name Size in Bytes Type of data When Present Notes
points_count 4 bytes int32
points 12 bytes * points_count float32[points_count][3]
points_parent_indices 4 bytes * points_count int32[points_count] only present if version is 8 or higher
skeleton bones
Name Size in Bytes Type of data When Present Notes
bones_count 4 bytes int32
bones_ends 8 bytes * bones_count int[bones_ends][2]
bones_parent_indices 4 bytes * bones_count int32[bones_count] only present if version is 8 or higher
bones_masses 4 bytes * bones_count float32[bones_count] only present if version is 6 or higher
bones_center_of_mass 12 bytes * bones_count float32[bones_count][3] only present if version is 7 or higher if version is less than 7 then calculate from bones_ends midpoints instead
bones_matrices 144 bytes * bones_count float32[bones_count][16] only present if version is 9 or higher this value is ignored and recalculated internally in engine regardless of version
the engine uses bones_ends to calculate it

PHXBN control joints

The skeleton control joints sections are only present if rigging_stage equals 1 (control_joints) or 2 (animate):

skeleton control joints header
Name Size in Bytes Type of data When Present Notes
vertices_count 4 bytes int32 only present if version is 11 or higher the actual value is always grabbed from the corresponding model file.
the engine double-checks that this value matches the model and triggers an error if it doesn't
control_joints_vertex_to_bone_weights 16 bytes * vertices_count float32[vertices_count][4]
control_joints_vertex_to_bone_ids 16 bytes * vertices_count float32[vertices_count][4] the values will always be integer values, but are encoded as float32
control_joints_hier_parents 4 bytes * bones_count int32[bones_count] the count of this array is BONE count not VERTEX count
TODO: What is hier_parents and how is it used?
control_joints_count 4 bytes int32

The next section is a list of variable size. count of list is control_joints_count.
The section is only present if rigging_stage equals 1 (control_joints) or 2 (animate).
Each list item will be from one of the following tables, and since the list isn't ordered by type, the amount of memory used for this table can't really be determined until the whole list is checked one item at a time:

skeleton hinge control joint
Name Size in Bytes Type of data When Present Notes
joint_type 4 bytes enum JointType (int32) value is 0
hinge_stop_angle 4 bytes * 2 float32[2] TODO: is this degrees or radians? what are the two dimensions?
bone_id 4 bytes * 2 int32[2]
hinge_axis 4 bytes * 3 float32[3]
skeleton amotor control joint
Name Size in Bytes Type of data When Present Notes
joint_type 4 bytes enum JointType (int32) value is 1
amotor_stop_angle 4 bytes * 6 float32[6] TODO: is this degrees or radians? what are the six dimensions?
bone_id 4 bytes * 2 int32[2]
skeleton fixed control joint
Name Size in Bytes Type of data When Present Notes
joint_type 4 bytes enum JointType (int32) value is 2
bone_id 4 bytes * 2 int32[2]

PHXBN simple IK bones

The skeleton simple ik bones sections are only present if version is 10 or higher:

skeleton simple ik bones header
Name Size in Bytes Type of data When Present Notes
simple_ik_bones_count 4 bytes int32 only present if version is 10 or higher

The next section is a list of variable size. count of list is simple_ik_bones_count.
The section is only present if version is 10 or higher.
Each list entry is variable size, based on the length of the bone name. Since each entry is variable size, the amount of memory used for this table can't really be determined until the whole list is checked one item at a time:

skeleton simple ik bones entries
Name Size in Bytes Type of data When Present Notes
bone_id 4 bytes int32
chain_length 4 bytes int32
name_length 4 bytes int32
name 1 byte * name_length char8[name_length]

PHXFZX

This is the bounding volumes format for ragdoll characters in the Phoenix engine.

Label Field:

The label for each object is a set of words. These words are tokens that specify the volume's flags, the volume type, and the specific body part name it applies to.

Label Field Flags:

  • L (flag as left version of body part)
  • R (flag as right version of body part)
  • Capsule (volume type)
  • Box (volume type)
  • Sphere (volume type)
  • M (flag as mirrored. This means that it will take the L or R, and create a second volume automatically, flipping it over the body's mirror plane)

Valid body part labels:

  • Eartip
  • Earbase
  • Head
  • Chest
  • Upperarm
  • Forearm
  • Abdomen
  • Hip
  • Thigh
  • Shin
  • Foot
  • Tail1
  • Tail2
  • Tail3
  • Tail4
  • Tail5
  • Tail6

The tokens are case-insensitive, and separated by spaces. If you use the flags correctly, the order doesn't matter.

If you specify conflicting flags, or multiple body parts, it will take the last token of that type. So you cannot specify more than one volume type per-object, both left and right for the same volume, etc.

Examples for the label field:

  • L Eartip Sphere M (note that because this is mirrored, there doesn't also need to be a R Eartip Sphere volume)
  • Abdomen Capsule
//------------------------------------------------
//--- 010 Editor v7.0.2 Binary Template
//
//      File: PHXFZX.bt
//   Authors: Wolfire Games
//   Version: 1.0
//   Purpose: Ragdoll Bounding Volumes for Phoenix Engine
//  Category: 3D
// File Mask: *.fzx
//   History:
//   1.0   Initial release
//------------------------------------------------

struct quat {
    float w;
    float x;
    float y;
    float z;
};

struct vec3 {
    float x;
    float y;
    float z;
};

// header
uchar fzx_header_mark;  // Must be character 211
char fzx_header_message[7]; // Must be 'F' 'Z' 'X' '\r' '\n' '<space>' '\n'
int32 version;  // Must be 1 (for now)
int32 object_count;

// body
struct FzxObject {
    int32 label_length;  // Must be 255 or less
    char label[label_length];  // Not null-terminated
    quat rotation;
    vec3 location;
    vec3 scale;
} objects[object_count] <optimize=false>;

// footer
int32 file_size;  // Must match actual file size in bytes, minus the entire footer (including this field)
char fzx_footer_message[3];  // Must be 'F' 'Z' 'X'

PHXSAVE

This is the save file format for the Phoenix engine.

Note: This is not the most up to date version of this format! This does not cover the .sav3 file type.

//------------------------------------------------
//--- 010 Editor v7.0.2 Binary Template
//
//      File: PHXSAVE.bt
//   Authors: Wolfire Games
//   Version: 1.0
//   Purpose: Script-persisted data for Phoenix Engine
//  Category: Games
// File Mask: *.sav
//  ID Bytes: 4F 76 65 72 67 72 6F 77 74 68 20 53 61 76 65
//   History:
//   1.0   Initial release
//------------------------------------------------

#define FILE_ID_LENGTH Strlen("Overgrowth Save")
#define MD5_LENGTH 16

typedef struct {  // PAIR
  uint16 key_length <fgcolor=cLtGray>;
  char key[key_length];

  uint16 value_length <fgcolor=cLtGray>;
  char value[value_length] <optimize=false>;
} PAIR <read=get_pair_value>;

string get_pair_name(PAIR& pair_data) {
  return pair_data.key;
}

string get_pair_value(PAIR& pair_data) {
  if(pair_data.value_length > 0) {
    return pair_data.value;
  } else {
    return "";
  }
}

typedef struct {  // MD5
  uchar value[MD5_LENGTH];
} MD5 <read=get_md5>;

string get_md5(MD5& md5_data) {
  char result[MD5_LENGTH * 2];
  int i;
  SPrintf(result, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x",
    md5_data.value[0], md5_data.value[1], md5_data.value[2], md5_data.value[3],
    md5_data.value[4], md5_data.value[5], md5_data.value[6], md5_data.value[7],
    md5_data.value[8], md5_data.value[9], md5_data.value[10], md5_data.value[11],
    md5_data.value[12], md5_data.value[13], md5_data.value[14], md5_data.value[15]);
  return result;
}

typedef struct {  // LEVEL_OR_MOD
  uint16 name_length <fgcolor=cLtGray>;
  char name[name_length] <fgcolor=cBlue>;

  MD5 md5 <name="md5", fgcolor=cGray>;

  uint16 pair_count <fgcolor=cLtGray>;
  PAIR pairs[pair_count] <optimize=false, name=get_pair_name>;
} LEVEL_OR_MOD;

string get_level_or_mod_name(LEVEL_OR_MOD& level_or_mod_data) {
  return level_or_mod_data.name;
}

struct {  // file
  char id_bytes[FILE_ID_LENGTH] <hidden=true, fgcolor=cLtGray>;
  uint16 file_version;

  uint16 level_or_mod_count <fgcolor=cLtGray>;
  LEVEL_OR_MOD level_or_mods[level_or_mod_count] <optimize=false, name=get_level_or_mod_name>;
} file;