Intravision/reactos
1
0
Fork 0
mirror of https://github.com/reactos/reactos.git synced 2024-07-29 15:49:03 +00:00
Code Issues Packages Projects Releases Wiki Activity
reactos/modules/rosapps/applications/devutils/btrfstools/btrfs_structures.py
Victor Perevertkin d0847939e4 [BTRFSTOOLS] Added python scripts for inspecting BTRFS filesystem 
internals
2018-08-20 08:26:56 +02:00

1293 lines
37 KiB
Python
Raw Blame History

# PROJECT: Python tools for traversing BTRFS structures
# LICENSE: GPL-2.0+ (https://spdx.org/licenses/GPL-2.0+)
# PURPOSE: Classes and structures for BTRFS on-disk layout
# COPYRIGHT: Copyright 2018 Victor Perevertkin (victor@perevertkin.ru)
# some code was taken from https://github.com/knorrie/python-btrfs
from btrfs_constants import *
import struct
from collections import namedtuple, OrderedDict
import collections.abc
import copy
import datetime
import os
import uuid
import crc32c
ULLONG_MAX = (1 << 64) - 1
ULONG_MAX = (1 << 32) - 1
def ULL(n):
return n & ULLONG_MAX
ROOT_TREE_OBJECTID = 1
EXTENT_TREE_OBJECTID = 2
CHUNK_TREE_OBJECTID = 3
DEV_TREE_OBJECTID = 4
FS_TREE_OBJECTID = 5
ROOT_TREE_DIR_OBJECTID = 6
CSUM_TREE_OBJECTID = 7
QUOTA_TREE_OBJECTID = 8
UUID_TREE_OBJECTID = 9
FREE_SPACE_TREE_OBJECTID = 10
DEV_STATS_OBJECTID = 0
BALANCE_OBJECTID = ULL(-4)
ORPHAN_OBJECTID = ULL(-5)
TREE_LOG_OBJECTID = ULL(-6)
TREE_LOG_FIXUP_OBJECTID = ULL(-7)
TREE_RELOC_OBJECTID = ULL(-8)
DATA_RELOC_TREE_OBJECTID = ULL(-9)
EXTENT_CSUM_OBJECTID = ULL(-10)
FREE_SPACE_OBJECTID = ULL(-11)
FREE_INO_OBJECTID = ULL(-12)
MULTIPLE_OBJECTIDS = ULL(-255)
FIRST_FREE_OBJECTID = 256
LAST_FREE_OBJECTID = ULL(-256)
FIRST_CHUNK_TREE_OBJECTID = 256
DEV_ITEMS_OBJECTID = 1
BTRFS_SYSTEM_CHUNK_ARRAY_SIZE = 2048
INODE_ITEM_KEY = 1
INODE_REF_KEY = 12
INODE_EXTREF_KEY = 13
XATTR_ITEM_KEY = 24
ORPHAN_ITEM_KEY = 48
DIR_LOG_ITEM_KEY = 60
DIR_LOG_INDEX_KEY = 72
DIR_ITEM_KEY = 84
DIR_INDEX_KEY = 96
EXTENT_DATA_KEY = 108
EXTENT_CSUM_KEY = 128
ROOT_ITEM_KEY = 132
ROOT_BACKREF_KEY = 144
ROOT_REF_KEY = 156
EXTENT_ITEM_KEY = 168
METADATA_ITEM_KEY = 169
TREE_BLOCK_REF_KEY = 176
EXTENT_DATA_REF_KEY = 178
SHARED_BLOCK_REF_KEY = 182
SHARED_DATA_REF_KEY = 184
BLOCK_GROUP_ITEM_KEY = 192
FREE_SPACE_INFO_KEY = 198
FREE_SPACE_EXTENT_KEY = 199
FREE_SPACE_BITMAP_KEY = 200
DEV_EXTENT_KEY = 204
DEV_ITEM_KEY = 216
CHUNK_ITEM_KEY = 228
QGROUP_STATUS_KEY = 240
QGROUP_INFO_KEY = 242
QGROUP_LIMIT_KEY = 244
QGROUP_RELATION_KEY = 246
BALANCE_ITEM_KEY = 248
DEV_STATS_KEY = 249
DEV_REPLACE_KEY = 250
UUID_KEY_SUBVOL = 251
UUID_KEY_RECEIVED_SUBVOL = 252
STRING_ITEM_KEY = 253
BLOCK_GROUP_SINGLE = 0
BLOCK_GROUP_DATA = 1 << 0
BLOCK_GROUP_SYSTEM = 1 << 1
BLOCK_GROUP_METADATA = 1 << 2
BLOCK_GROUP_RAID0 = 1 << 3
BLOCK_GROUP_RAID1 = 1 << 4
BLOCK_GROUP_DUP = 1 << 5
BLOCK_GROUP_RAID10 = 1 << 6
BLOCK_GROUP_RAID5 = 1 << 7
BLOCK_GROUP_RAID6 = 1 << 8
BLOCK_GROUP_TYPE_MASK = (
BLOCK_GROUP_DATA |
BLOCK_GROUP_SYSTEM |
BLOCK_GROUP_METADATA
)
BLOCK_GROUP_PROFILE_MASK = (
BLOCK_GROUP_RAID0 |
BLOCK_GROUP_RAID1 |
BLOCK_GROUP_RAID5 |
BLOCK_GROUP_RAID6 |
BLOCK_GROUP_DUP |
BLOCK_GROUP_RAID10
)
AVAIL_ALLOC_BIT_SINGLE = 1 << 48 # used in balance_args
SPACE_INFO_GLOBAL_RSV = 1 << 49
_block_group_flags_str_map = {
BLOCK_GROUP_DATA: 'DATA',
BLOCK_GROUP_METADATA: 'METADATA',
BLOCK_GROUP_SYSTEM: 'SYSTEM',
BLOCK_GROUP_RAID0: 'RAID0',
BLOCK_GROUP_RAID1: 'RAID1',
BLOCK_GROUP_DUP: 'DUP',
BLOCK_GROUP_RAID10: 'RAID10',
BLOCK_GROUP_RAID5: 'RAID5',
BLOCK_GROUP_RAID6: 'RAID6',
}
_balance_args_profiles_str_map = {
BLOCK_GROUP_RAID0: 'RAID0',
BLOCK_GROUP_RAID1: 'RAID1',
BLOCK_GROUP_DUP: 'DUP',
BLOCK_GROUP_RAID10: 'RAID10',
BLOCK_GROUP_RAID5: 'RAID5',
BLOCK_GROUP_RAID6: 'RAID6',
AVAIL_ALLOC_BIT_SINGLE: 'SINGLE',
}
QGROUP_LEVEL_SHIFT = 48
EXTENT_FLAG_DATA = 1 << 0
EXTENT_FLAG_TREE_BLOCK = 1 << 1
BLOCK_FLAG_FULL_BACKREF = 1 << 8
_extent_flags_str_map = {
EXTENT_FLAG_DATA: 'DATA',
EXTENT_FLAG_TREE_BLOCK: 'TREE_BLOCK',
BLOCK_FLAG_FULL_BACKREF: 'FULL_BACKREF',
}
INODE_NODATASUM = 1 << 0
INODE_NODATACOW = 1 << 1
INODE_READONLY = 1 << 2
INODE_NOCOMPRESS = 1 << 3
INODE_PREALLOC = 1 << 4
INODE_SYNC = 1 << 5
INODE_IMMUTABLE = 1 << 6
INODE_APPEND = 1 << 7
INODE_NODUMP = 1 << 8
INODE_NOATIME = 1 << 9
INODE_DIRSYNC = 1 << 10
INODE_COMPRESS = 1 << 11
_inode_flags_str_map = {
INODE_NODATASUM: 'NODATASUM',
INODE_READONLY: 'READONLY',
INODE_NOCOMPRESS: 'NOCOMPRESS',
INODE_PREALLOC: 'PREALLOC',
INODE_SYNC: 'SYNC',
INODE_IMMUTABLE: 'IMMUTABLE',
INODE_APPEND: 'APPEND',
INODE_NODUMP: 'NODUMP',
INODE_NOATIME: 'NOATIME',
INODE_DIRSYNC: 'DIRSYNC',
INODE_COMPRESS: 'COMPRESS',
}
ROOT_SUBVOL_RDONLY = 1 << 0
_root_flags_str_map = {
ROOT_SUBVOL_RDONLY: 'RDONLY',
}
FT_UNKNOWN = 0
FT_REG_FILE = 1
FT_DIR = 2
FT_CHRDEV = 3
FT_BLKDEV = 4
FT_FIFO = 5
FT_SOCK = 6
FT_SYMLINK = 7
FT_XATTR = 8
FT_MAX = 9
_dir_item_type_str_map = {
FT_UNKNOWN: 'UNKNOWN',
FT_REG_FILE: 'FILE',
FT_DIR: 'DIR',
FT_CHRDEV: 'CHRDEV',
FT_BLKDEV: 'BLKDEV',
FT_FIFO: 'FIFO',
FT_SOCK: 'SOCK',
FT_SYMLINK: 'SYMLINK',
FT_XATTR: 'XATTR',
}
COMPRESS_NONE = 0
COMPRESS_ZLIB = 1
COMPRESS_LZO = 2
COMPRESS_ZSTD = 3
_compress_type_str_map = {
COMPRESS_NONE: 'none',
COMPRESS_ZLIB: 'zlib',
COMPRESS_LZO: 'lzo',
COMPRESS_ZSTD: 'zstd',
}
FILE_EXTENT_INLINE = 0
FILE_EXTENT_REG = 1
FILE_EXTENT_PREALLOC = 2
_file_extent_type_str_map = {
FILE_EXTENT_INLINE: 'inline',
FILE_EXTENT_REG: 'regular',
FILE_EXTENT_PREALLOC: 'prealloc',
}
def qgroup_level(objectid):
return objectid >> QGROUP_LEVEL_SHIFT
def qgroup_subvid(objectid):
return objectid & ((1 << QGROUP_LEVEL_SHIFT) - 1)
_key_objectid_str_map = {
ROOT_TREE_OBJECTID: 'ROOT_TREE',
EXTENT_TREE_OBJECTID: 'EXTENT_TREE',
CHUNK_TREE_OBJECTID: 'CHUNK_TREE',
DEV_TREE_OBJECTID: 'DEV_TREE',
FS_TREE_OBJECTID: 'FS_TREE',
ROOT_TREE_DIR_OBJECTID: 'ROOT_TREE_DIR',
CSUM_TREE_OBJECTID: 'CSUM_TREE',
QUOTA_TREE_OBJECTID: 'QUOTA_TREE',
UUID_TREE_OBJECTID: 'UUID_TREE',
FREE_SPACE_TREE_OBJECTID: 'FREE_SPACE_TREE',
BALANCE_OBJECTID: 'BALANCE',
ORPHAN_OBJECTID: 'ORPHAN',
TREE_LOG_OBJECTID: 'TREE_LOG',
TREE_LOG_FIXUP_OBJECTID: 'TREE_LOG_FIXUP',
TREE_RELOC_OBJECTID: 'TREE_RELOC',
DATA_RELOC_TREE_OBJECTID: 'DATA_RELOC_TREE',
EXTENT_CSUM_OBJECTID: 'EXTENT_CSUM',
FREE_SPACE_OBJECTID: 'FREE_SPACE',
FREE_INO_OBJECTID: 'FREE_INO',
MULTIPLE_OBJECTIDS: 'MULTIPLE',
}
def key_objectid_str(objectid, _type):
if _type == DEV_EXTENT_KEY:
return str(objectid)
if _type == QGROUP_RELATION_KEY:
return "{}/{}".format(qgroup_level(objectid), qgroup_subvid(objectid))
if _type == UUID_KEY_SUBVOL or _type == UUID_KEY_RECEIVED_SUBVOL:
return "0x{:0>16x}".format(objectid)
if objectid == ROOT_TREE_OBJECTID and _type == DEV_ITEM_KEY:
return 'DEV_ITEMS'
if objectid == DEV_STATS_OBJECTID and _type == DEV_STATS_KEY:
return 'DEV_STATS'
if objectid == FIRST_CHUNK_TREE_OBJECTID and _type == CHUNK_ITEM_KEY:
return 'FIRST_CHUNK_TREE'
if objectid == ULLONG_MAX:
return '-1'
return _key_objectid_str_map.get(objectid, str(objectid))
_key_type_str_map = {
INODE_ITEM_KEY: 'INODE_ITEM',
INODE_REF_KEY: 'INODE_REF',
INODE_EXTREF_KEY: 'INODE_EXTREF',
XATTR_ITEM_KEY: 'XATTR_ITEM',
ORPHAN_ITEM_KEY: 'ORPHAN_ITEM',
DIR_LOG_ITEM_KEY: 'DIR_LOG_ITEM',
DIR_LOG_INDEX_KEY: 'DIR_LOG_INDEX',
DIR_ITEM_KEY: 'DIR_ITEM',
DIR_INDEX_KEY: 'DIR_INDEX',
EXTENT_DATA_KEY: 'EXTENT_DATA',
EXTENT_CSUM_KEY: 'EXTENT_CSUM',
ROOT_ITEM_KEY: 'ROOT_ITEM',
ROOT_BACKREF_KEY: 'ROOT_BACKREF',
ROOT_REF_KEY: 'ROOT_REF',
EXTENT_ITEM_KEY: 'EXTENT_ITEM',
METADATA_ITEM_KEY: 'METADATA_ITEM',
TREE_BLOCK_REF_KEY: 'TREE_BLOCK_REF',
EXTENT_DATA_REF_KEY: 'EXTENT_DATA_REF',
SHARED_BLOCK_REF_KEY: 'SHARED_BLOCK_REF',
SHARED_DATA_REF_KEY: 'SHARED_DATA_REF',
BLOCK_GROUP_ITEM_KEY: 'BLOCK_GROUP_ITEM',
FREE_SPACE_INFO_KEY: 'FREE_SPACE_INFO',
FREE_SPACE_EXTENT_KEY: 'FREE_SPACE_EXTENT',
FREE_SPACE_BITMAP_KEY: 'FREE_SPACE_BITMAP',
DEV_EXTENT_KEY: 'DEV_EXTENT',
DEV_ITEM_KEY: 'DEV_ITEM',
CHUNK_ITEM_KEY: 'CHUNK_ITEM',
QGROUP_STATUS_KEY: 'QGROUP_STATUS',
QGROUP_INFO_KEY: 'QGROUP_INFO',
QGROUP_LIMIT_KEY: 'QGROUP_LIMIT',
QGROUP_RELATION_KEY: 'QGROUP_RELATION',
BALANCE_ITEM_KEY: 'BALANCE_ITEM',
DEV_STATS_KEY: 'DEV_STATS',
DEV_REPLACE_KEY: 'DEV_REPLACE',
UUID_KEY_SUBVOL: 'UUID_SUBVOL',
UUID_KEY_RECEIVED_SUBVOL: 'RECEIVED_SUBVOL',
STRING_ITEM_KEY: 'STRING_ITEM',
}
# === Helper functions
def key_type_str(_type):
return _key_type_str_map.get(_type, str(_type))
def key_offset_str(offset, _type):
if _type == QGROUP_RELATION_KEY or _type == QGROUP_INFO_KEY or _type == QGROUP_LIMIT_KEY:
return "{}/{}".format(qgroup_level(offset), qgroup_subvid(offset))
if _type == UUID_KEY_SUBVOL or _type == UUID_KEY_RECEIVED_SUBVOL:
return "0x{:0>16x}".format(offset)
if _type == ROOT_ITEM_KEY:
return _key_objectid_str_map.get(offset, str(offset))
if offset == ULLONG_MAX:
return '-1'
return str(offset)
def flags_str(flags, flags_str_map):
ret = []
for flag in sorted(flags_str_map.keys()):
if flags & flag:
ret.append(flags_str_map[flag])
if len(ret) == 0:
ret.append("none")
return '|'.join(ret)
def embedded_text_for_str(text):
try:
return "utf-8 {}".format(text.decode('utf-8'))
except UnicodeDecodeError:
return "raw {}".format(repr(text))
# === Basic structures
class TimeSpec(object):
sstruct = struct.Struct('<QL')
@staticmethod
def from_values(sec, nsec):
t = TimeSpec.__new__(TimeSpec)
t.sec = sec
t.nsec = nsec
return t
def __init__(self, data):
self.sec, self.nsec = TimeSpec.sstruct.unpack_from(data)
@property
def iso8601(self):
return datetime.datetime.utcfromtimestamp(
float("{self.sec}.{self.nsec}".format(self=self))
).isoformat()
def __str__(self):
return "{self.sec}.{self.nsec} ({self.iso8601})".format(self=self)
class Key(object):
def __init__(self, objectid, _type, offset):
self._objectid = objectid
self._type = _type
self._offset = offset
self._pack()
@property
def objectid(self):
return self._objectid
@objectid.setter
def objectid(self, _objectid):
self._objectid = _objectid
self._pack()
@property
def type(self):
return self._type
@type.setter
def type(self, _type):
self._type = _type
self._pack()
@property
def offset(self):
return self._offset
@offset.setter
def offset(self, _offset):
self._offset = _offset
self._pack()
@property
def key(self):
return self._key
@key.setter
def key(self, _key):
self._key = _key
self._unpack()
def _pack(self):
self._key = (self.objectid << 72) + (self._type << 64) + self.offset
def _unpack(self):
self._objectid = self._key >> 72
self._type = (self._key & ((1 << 72) - 1)) >> 64
self._offset = (self._key & ((1 << 64) - 1))
def __lt__(self, other):
if isinstance(other, Key):
return self._key < other._key
return self._key < other
def __le__(self, other):
if isinstance(other, Key):
return self._key <= other._key
return self._key <= other
def __eq__(self, other):
if isinstance(other, Key):
return self._key == other._key
return self._key == other
def __ge__(self, other):
if isinstance(other, Key):
return self._key >= other._key
return self._key >= other
def __gt__(self, other):
if isinstance(other, Key):
return self._key > other._key
return self._key > other
def __str__(self):
return "({} {} {})".format(
key_objectid_str(self._objectid, self._type),
key_type_str(self._type),
key_offset_str(self._offset, self._type),
)
def __add__(self, amount):
new_key = copy.copy(self)
new_key.key += amount
return new_key
def __sub__(self, amount):
new_key = copy.copy(self)
new_key.key -= amount
return new_key
class DiskKey(Key):
sstruct = struct.Struct('<QBQ')
def __init__(self, data):
super(DiskKey, self).__init__(*DiskKey.sstruct.unpack_from(data))
class InnerKey(Key):
sstruct = struct.Struct('<QBQQQ')
def __init__(self, data):
unpacked_data = InnerKey.sstruct.unpack_from(data)
super().__init__(*unpacked_data[:3])
self.block_num = unpacked_data[3]
self.generation = unpacked_data[4]
def __str__(self):
return "(inner_key {} {} {} block_num {}, generation {})".format(
key_objectid_str(self._objectid, self._type),
key_type_str(self._type),
key_offset_str(self._offset, self._type),
self.block_num,
self.generation,
)
class LeafKey(Key):
sstruct = struct.Struct('<QBQLL')
def __init__(self, data):
unpacked_data = LeafKey.sstruct.unpack_from(data)
super().__init__(*unpacked_data[:3])
self.data_offset = unpacked_data[3]
self.data_size = unpacked_data[4]
def __str__(self):
return "(leaf_key {} {} {} data_offset {:#x} data_size {})".format(
key_objectid_str(self._objectid, self._type),
key_type_str(self._type),
key_offset_str(self._offset, self._type),
self.data_offset,
self.data_size,
)
class ItemData(object):
def __init__(self, key):
self.key = key
def setattr_from_key(self, objectid_attr=None, type_attr=None, offset_attr=None):
if objectid_attr is not None:
setattr(self, objectid_attr, self.key.objectid)
if type_attr is not None:
setattr(self, type_attr, self.key.type)
if offset_attr is not None:
setattr(self, offset_attr, self.key.offset)
self._key_attrs = objectid_attr, type_attr, offset_attr
@property
def key_attrs(self):
try:
return self._key_attrs
except AttributeError:
return None, None, None
def __lt__(self, other):
return self.key < other.key
superblock = struct.Struct('<32x16s2Q8s9Q5L4QH2B611x2048s')
# NOTE: the structure is not complete
# FS UUID
# Physical block address
# Flags
# Signature (_BHRfS_M)
# generation
# Log. address of root of tree roots
# Log. address of chunk tree root
# Log. address of log tree root
# log_root_transid
# total_bytes
# bytes_used
# root_dir_objectid (usually 6)
# num_devices
# sectorsize
# nodesize
# __unused_leafsize
# stripesize
# sys_chunk_array_size
# chunk_root_generation
# compat_flags
# compat_ro_flags
# incompat_flags
# csum_type
# root_level 23
# chunk_root_level 24
# ---
# sys_chunk_array
_node_header_struct = struct.Struct('<32x16sQQ16sQQLB')
NodeHeader = namedtuple('NodeHeader', 'FS_UUID node_addr flags chunk_tree_uuid generation tree_id items_num level')
# === Items
class InodeItem(ItemData):
_inode_item = [
struct.Struct('<5Q4L3Q32x'),
TimeSpec.sstruct,
TimeSpec.sstruct,
TimeSpec.sstruct,
TimeSpec.sstruct,
]
sstruct = struct.Struct('<' + ''.join([s.format[1:].decode() for s in _inode_item]))
def __init__(self, key, data):
super().__init__(key)
self.generation, self.transid, self.size, self.nbytes, self.block_group, \
self.nlink, self.uid, self.gid, self.mode, self.rdev, self.flags, self.sequence = \
InodeItem._inode_item[0].unpack_from(data)
pos = InodeItem._inode_item[0].size
next_pos = pos + TimeSpec.sstruct.size
self.atime = TimeSpec(data[pos:next_pos])
pos, next_pos = next_pos, next_pos + TimeSpec.sstruct.size
self.ctime = TimeSpec(data[pos:next_pos])
pos, next_pos = next_pos, next_pos + TimeSpec.sstruct.size
self.mtime = TimeSpec(data[pos:next_pos])
pos, next_pos = next_pos, next_pos + TimeSpec.sstruct.size
self.otime = TimeSpec(data[pos:next_pos])
@property
def flags_str(self):
return flags_str(self.flags, _inode_flags_str_map)
def __str__(self):
return "inode generation {self.generation} transid {self.transid} size {self.size} " \
"nbytes {self.nbytes} block_group {self.block_group} mode {self.mode:05o} " \
"nlink {self.nlink} uid {self.uid} gid {self.gid} rdev {self.rdev} " \
"flags {self.flags:#x}({self.flags_str})".format(self=self)
class RootItem(ItemData):
_root_item = [
InodeItem.sstruct,
struct.Struct('<7QL'),
DiskKey.sstruct,
struct.Struct('<BBQ16s16s16s4Q'),
TimeSpec.sstruct,
TimeSpec.sstruct,
TimeSpec.sstruct,
TimeSpec.sstruct,
]
sstruct = struct.Struct('<' + ''.join([s.format[1:].decode() for s in _root_item]))
def __init__(self, key, data):
super().__init__(key)
self.inode = InodeItem(None, data[:InodeItem.sstruct.size])
pos = InodeItem.sstruct.size
self.generation, self.dirid, self.bytenr, self.byte_limit, self.bytes_used, \
self.last_snapshot, self.flags, self.refs = \
RootItem._root_item[1].unpack_from(data, pos)
pos += RootItem._root_item[1].size
self.drop_progress = DiskKey(data[pos:pos+DiskKey.sstruct.size])
pos += DiskKey.sstruct.size
self.drop_level, self.level, self.generation_v2, uuid_bytes, parent_uuid_bytes, \
received_uuid_bytes, self.ctransid, self.otransid, self.stransid, self.rtransid = \
RootItem._root_item[3].unpack_from(data, pos)
self.uuid = uuid.UUID(bytes=uuid_bytes)
self.parent_uuid = uuid.UUID(bytes=parent_uuid_bytes)
self.received_uuid = uuid.UUID(bytes=received_uuid_bytes)
pos += RootItem._root_item[3].size
next_pos = pos + TimeSpec.sstruct.size
self.ctime = TimeSpec(data[pos:next_pos])
pos, next_pos = next_pos, next_pos + TimeSpec.sstruct.size
self.otime = TimeSpec(data[pos:next_pos])
pos, next_pos = next_pos, next_pos + TimeSpec.sstruct.size
self.stime = TimeSpec(data[pos:next_pos])
pos, next_pos = next_pos, next_pos + TimeSpec.sstruct.size
self.rtime = TimeSpec(data[pos:next_pos])
@property
def flags_str(self):
return flags_str(self.flags, _root_flags_str_map)
def __str__(self):
return "root {self.key.objectid} uuid {self.uuid} " \
"generation {self.generation} last_snapshot {self.last_snapshot} " \
"bytenr {self.bytenr:#x} level {self.level} " \
"flags {self.flags:#x}({self.flags_str})".format(self=self)
class Chunk(ItemData):
sstruct = struct.Struct('<4Q3L2H')
def __init__(self, key, data):
super().__init__(key)
self.setattr_from_key(offset_attr='vaddr')
self.length, self.owner, self.stripe_len, self.type, self.io_align, \
self.io_width, self.sector_size, self.num_stripes, self.sub_stripes = \
Chunk.sstruct.unpack_from(data)
self.stripes = []
pos = Chunk.sstruct.size
for i in range(self.num_stripes):
next_pos = pos + Stripe.sstruct.size
self.stripes.append(Stripe(data[pos:next_pos]))
pos = next_pos
@property
def size(self):
return Chunk.sstruct.size + self.num_stripes * Stripe.sstruct.size
@property
def type_str(self):
return flags_str(self.type, _block_group_flags_str_map)
def __str__(self):
return "chunk vaddr {self.vaddr:#x} type {self.type_str} length {self.length} " \
"num_stripes {self.num_stripes}".format(self=self)
class Stripe(object):
sstruct = struct.Struct('<2Q16s')
def __init__(self, data):
self.devid, self.offset, uuid_bytes = Stripe.sstruct.unpack(data)
self.uuid = uuid.UUID(bytes=uuid_bytes)
def __str__(self):
return "stripe devid {self.devid} offset {self.offset:#x}".format(self=self)
class InodeRefList(ItemData, collections.abc.MutableSequence):
def __init__(self, header, data):
super().__init__(header)
self._list = []
pos = 0
while pos < header.len:
inode_ref = InodeRef(data, pos)
self._list.append(inode_ref)
pos += len(inode_ref)
def __getitem__(self, index):
return self._list[index]
def __setitem__(self, index, value):
self._list[index] = value
def __delitem__(self, index):
del self._list[index]
def __len__(self):
return len(self._list)
def insert(self, index, value):
self._list.insert(index, value)
def __str__(self):
return "inode ref list size {}".format(len(self))
class InodeRef(ItemData):
sstruct = struct.Struct('<QH')
def __init__(self, key, data):
super().__init__(key)
self.index, self.name_len = InodeRef.sstruct.unpack_from(data)
self.name, = struct.Struct('<{}s'.format(self.name_len)).unpack_from(data, InodeRef.sstruct.size)
self._len = InodeRef.sstruct.size + self.name_len
@property
def name_str(self):
return embedded_text_for_str(self.name)
def __len__(self):
return self._len
def __str__(self):
return "inode ref index {self.index} name {self.name_str}".format(self=self)
class DirItemList(ItemData, collections.abc.MutableSequence):
def __init__(self, key, data):
super().__init__(key)
self._list = []
pos = 0
while pos < key.data_size:
cls = {DIR_ITEM_KEY: DirItem, XATTR_ITEM_KEY: XAttrItem}
dir_item = cls[self.key.type](data, pos)
self._list.append(dir_item)
pos += len(dir_item)
def __getitem__(self, index):
return self._list[index]
def __setitem__(self, index, value):
self._list[index] = value
def __delitem__(self, index):
del self._list[index]
def __len__(self):
return len(self._list)
def insert(self, index, value):
self._list.insert(index, value)
def __str__(self):
return "dir item list hash {self.key.offset} size {}".format(len(self), self=self)
class XAttrItemList(DirItemList):
def __str__(self):
return "xattr item list hash {self.key.offset} size {}".format(len(self), self=self)
class DirItem(object):
_dir_item = [
DiskKey.sstruct,
struct.Struct('<QHHB')
]
sstruct = struct.Struct('<' + ''.join([s.format[1:].decode() for s in _dir_item]))
def __init__(self, data, pos):
next_pos = pos + DiskKey.sstruct.size
self.location = DiskKey(data[pos:next_pos])
pos = next_pos
self.transid, self.data_len, self.name_len, self.type = \
DirItem._dir_item[1].unpack_from(data, pos)
pos += DirItem._dir_item[1].size
self.name, = struct.Struct('<{}s'.format(self.name_len)).unpack_from(data, pos)
pos += self.name_len
self.data, = struct.Struct('<{}s'.format(self.data_len)).unpack_from(data, pos)
pos += self.data_len
self._len = DirItem.sstruct.size + self.name_len + self.data_len
@property
def type_str(self):
return _dir_item_type_str_map[self.type]
@property
def name_str(self):
return embedded_text_for_str(self.name)
@property
def data_str(self):
return embedded_text_for_str(self.data)
def __len__(self):
return self._len
def __str__(self):
return "dir item location {self.location} type {self.type_str} " \
"name {self.name_str}".format(self=self)
class XAttrItem(DirItem):
def __str__(self):
return "xattr item name {self.name_str} data {self.data_str}".format(self=self)
class DirIndex(ItemData):
def __init__(self, header, data):
super().__init__(header)
self.location = DiskKey(data[:DiskKey.sstruct.size])
pos = DiskKey.sstruct.size
self.transid, self.data_len, self.name_len, self.type = \
DirItem._dir_item[1].unpack_from(data, pos)
pos += DirItem._dir_item[1].size
self.name, = struct.Struct('<{}s'.format(self.name_len)).unpack_from(data, pos)
@property
def type_str(self):
return _dir_item_type_str_map[self.type]
@property
def name_str(self):
return embedded_text_for_str(self.name)
def __str__(self):
return "dir index {self.key.offset} location {self.location} type {self.type_str} " \
"name {self.name_str}".format(self=self)
class FileExtentItem(ItemData):
_file_extent_item = [
struct.Struct('<QQBB2xB'),
struct.Struct('<4Q'),
]
sstruct = struct.Struct('<' + ''.join([s.format[1:].decode()
for s in _file_extent_item]))
def __init__(self, key, data):
super().__init__(key)
self.logical_offset = key.offset
self.generation, self.ram_bytes, self.compression, self.encryption, self.type = \
FileExtentItem._file_extent_item[0].unpack_from(data)
if self.type != FILE_EXTENT_INLINE:
# These are confusing, so they deserve a comment in the code:
# (disk_bytenr EXTENT_ITEM disk_num_bytes) is the tree key of
# the extent item storing the actual data.
#
# The third one, offset is the offset inside that extent where the
# data we need starts. num_bytes is the amount of bytes to be used
# from that offset onwards.
#
# Remember that these numbers always be multiples of disk block
# sizes, because that's how it gets cowed. We don't just use 1 or 2
# bytes from another extent.
pos = FileExtentItem._file_extent_item[0].size
self.disk_bytenr, self.disk_num_bytes, self.offset, self.num_bytes = \
FileExtentItem._file_extent_item[1].unpack_from(data, pos)
else:
self._inline_encoded_nbytes = key.data_size - FileExtentItem._file_extent_item[0].size
@property
def compression_str(self):
return _compress_type_str_map.get(self.compression, 'unknown')
@property
def type_str(self):
return _file_extent_type_str_map.get(self.type, 'unknown')
def __str__(self):
ret = ["extent data at {self.logical_offset} generation {self.generation} "
"ram_bytes {self.ram_bytes} "
"compression {self.compression_str} type {self.type_str}".format(self=self)]
if self.type != FILE_EXTENT_INLINE:
ret.append("disk_bytenr {self.disk_bytenr} disk_num_bytes {self.disk_num_bytes} "
"offset {self.offset} num_bytes {self.num_bytes}".format(self=self))
else:
ret.append("inline_encoded_nbytes {self._inline_encoded_nbytes}".format(self=self))
return ' '.join(ret)
class ExtentItem(ItemData):
sstruct = struct.Struct('<3Q')
extent_inline_ref = struct.Struct('<BQ')
def __init__(self, header, data, load_data_refs=True, load_metadata_refs=True):
super().__init__(header)
self.setattr_from_key(objectid_attr='vaddr', offset_attr='length')
pos = 0
self.refs, self.generation, self.flags = ExtentItem.sstruct.unpack_from(data, pos)
pos += ExtentItem.sstruct.size
if self.flags == EXTENT_FLAG_DATA and load_data_refs:
self.extent_data_refs = []
self.shared_data_refs = []
while pos < len(data):
inline_ref_type, inline_ref_offset = \
ExtentItem.extent_inline_ref.unpack_from(data, pos)
if inline_ref_type == EXTENT_DATA_REF_KEY:
pos += 1
next_pos = pos + InlineExtentDataRef.sstruct.size
self.extent_data_refs.append(InlineExtentDataRef(data[pos:next_pos]))
pos = next_pos
elif inline_ref_type == SHARED_DATA_REF_KEY:
pos += 1
next_pos = pos + InlineSharedDataRef.inline_shared_data_ref.size
self.shared_data_refs.append(InlineSharedDataRef(data[pos:next_pos]))
pos = next_pos
elif self.flags & EXTENT_FLAG_TREE_BLOCK and load_metadata_refs:
next_pos = pos + TreeBlockInfo.tree_block_info.size
self.tree_block_info = TreeBlockInfo(data[pos:next_pos])
pos = next_pos
self.tree_block_refs = []
self.shared_block_refs = []
while pos < len(data):
inline_ref_type, inline_ref_offset = \
ExtentItem.extent_inline_ref.unpack_from(data, pos)
if inline_ref_type == TREE_BLOCK_REF_KEY:
self.tree_block_refs.append(InlineTreeBlockRef(inline_ref_offset))
elif inline_ref_type == SHARED_BLOCK_REF_KEY:
self.shared_block_refs.append(InlineSharedBlockRef(inline_ref_offset))
else:
raise Exception("BUG: expected inline TREE_BLOCK_REF or SHARED_BLOCK_REF_KEY "
"but got inline_ref_type {}".format(inline_ref_type))
pos += ExtentItem.extent_inline_ref.size
def append_extent_data_ref(self, ref):
self.extent_data_refs.append(ref)
def append_shared_data_ref(self, ref):
self.shared_data_refs.append(ref)
def append_tree_block_ref(self, ref):
self.tree_block_refs.append(ref)
def append_shared_block_ref(self, ref):
self.shared_block_refs.append(ref)
@property
def flags_str(self):
return flags_str(self.flags, _extent_flags_str_map)
def __str__(self):
return "extent vaddr {self.vaddr} length {self.length} refs {self.refs} " \
"gen {self.generation} flags {self.flags_str}".format(self=self)
class ExtentDataRef(ItemData):
sstruct = struct.Struct('<3QL')
def __init__(self, header, data):
super().__init__(header)
self.root, self.objectid, self.offset, self.count = \
ExtentDataRef.sstruct.unpack(data)
def __str__(self):
return "extent data backref root {self.root} objectid {self.objectid} " \
"offset {self.offset} count {self.count}".format(self=self)
class InlineExtentDataRef(ExtentDataRef):
sstruct = ExtentDataRef.sstruct
def __init__(self, data):
self.root, self.objectid, self.offset, self.count = \
InlineExtentDataRef.sstruct.unpack(data)
def __str__(self):
return "inline extent data backref root {self.root} objectid {self.objectid} " \
"offset {self.offset} count {self.count}".format(self=self)
class SharedDataRef(ItemData):
sstruct = struct.Struct('<L')
def __init__(self, header, data):
super().__init__(header)
self.setattr_from_key(offset_attr='parent')
self.count, = SharedDataRef.sstruct.unpack(data)
def __str__(self):
return "shared data backref parent {self.parent} count {self.count}".format(self=self)
class InlineSharedDataRef(SharedDataRef):
sstruct = struct.Struct('<QL')
def __init__(self, data):
self.parent, self.count = InlineSharedDataRef.sstruct.unpack(data)
def __str__(self):
return "inline shared data backref parent {self.parent} " \
"count {self.count}".format(self=self)
class TreeBlockInfo(object):
sstruct = struct.Struct('<QBQB')
def __init__(self, data):
tb_objectid, tb_type, tb_offset, self.level = \
TreeBlockInfo.sstruct.unpack(data)
self.key = Key(tb_objectid, tb_type, tb_offset)
def __str__(self):
return "tree block key {self.key} level {self.level}".format(self=self)
class TreeBlockRef(ItemData):
def __init__(self, header):
super().__init__(header)
self.setattr_from_key(offset_attr='root')
def __str__(self):
return "tree block backref root {}".format(key_objectid_str(self.root, None))
class InlineTreeBlockRef(TreeBlockRef):
def __init__(self, root):
self.root = root
def __str__(self):
return "inline tree block backref root {}".format(key_objectid_str(self.root, None))
class SharedBlockRef(ItemData):
def __init__(self, header):
super().__init__(header)
self.setattr_from_key(offset_attr='parent')
def __str__(self):
return "shared block backref parent {}".format(self.parent)
class InlineSharedBlockRef(SharedBlockRef):
def __init__(self, parent):
self.parent = parent
def __str__(self):
return "inline shared block backref parent {}".format(self.parent)
# === Main FileSystem class
def key_bin_search(fd, base_offset, item_size, cmp_item, min, max):
low = min
high = max
while low < high:
mid = (low + high) // 2
offset = base_offset + mid * item_size
fd.seek(offset)
key1 = DiskKey(fd.read(item_size))
if key1 > cmp_item:
high = mid
elif key1 < cmp_item:
low = mid + 1
else:
return True, mid
return False, low
chunk_map_item = namedtuple('chunk_map_item', 'logical physical length devid')
class FileSystem(object):
def __init__(self, path, part_offset):
self._chunk_map = OrderedDict()
self.path = path
self.part_offset = part_offset
self.fd = open(path, 'rb')
self.fd.seek(part_offset + 0x10000) # going to superblock
sb_bytes = self.fd.read(superblock.size)
sb_tuple = superblock.unpack(sb_bytes)
if sb_tuple[3] != b'_BHRfS_M':
raise "No signature found"
# setting base FS information
self.fsid = sb_tuple[0]
self.nodesize = sb_tuple[14]
self.sectorsize = sb_tuple[13]
self._chunk_root = sb_tuple[6]
self._chunk_root_level = sb_tuple[24]
self._tree_roots_root = sb_tuple[5]
self._tree_roots_root_level = sb_tuple[23]
# setting chunk map
sys_chunk_array_size = sb_tuple[17]
sys_chunk = sb_tuple[25][:sys_chunk_array_size]
pos = 0
while pos < sys_chunk_array_size:
key = DiskKey(sys_chunk[pos:])
pos += DiskKey.sstruct.size
chunk = Chunk(key, sys_chunk[pos:])
for st in chunk.stripes:
self._insert_chunk(chunk_map_item(chunk.vaddr, st.offset, chunk.length, st.devid))
pos += chunk.size
# setting tree roots
_, fs_tree_root_item = self.search_tree(self._tree_roots_root_level, self._tree_roots_root, Key(FS_TREE_OBJECTID, ROOT_ITEM_KEY, 0))
_, extent_tree_root_item = self.search_tree(self._tree_roots_root_level, self._tree_roots_root, Key(EXTENT_TREE_OBJECTID, ROOT_ITEM_KEY, 0))
self._fs_root_level = fs_tree_root_item.level
self._fs_root = fs_tree_root_item.bytenr
self._extent_root_level = extent_tree_root_item.level
self._extent_root = extent_tree_root_item.bytenr
@property
def chunk_root(self):
return self._chunk_root_level, self._chunk_root
@property
def tree_roots_root(self):
return self._tree_roots_root_level, self._tree_roots_root
@property
def fs_root(self):
return self._fs_root_level, self._fs_root
@property
def extent_root(self):
return self._extent_root_level, self._extent_root
def logical_to_physical(self, log):
cur_logical = next(iter(self._chunk_map)) # first item
for logical, cmi in self._chunk_map.items():
if logical > log:
break
cur_logical = logical
# if there is no address in chunk_map, searching in chunk_tree
if cur_logical + self._chunk_map[cur_logical].length < log:
def process_func(header, offset):
for i in range(header.items_num):
self.fd.seek(offset + i * LeafKey.sstruct.size)
k = LeafKey(self.fd.read(LeafKey.sstruct.size))
self.fd.seek(offset + k.data_offset)
if k.type == CHUNK_ITEM_KEY:
item = _key_type_class_map[k.type](k, self.fd.read(k.data_size))
for st in item.stripes:
self._insert_chunk(chunk_map_item(item.vaddr, st.offset, item.length, st.devid))
self.search_tree(self._chunk_root_level, self._chunk_root, Key(FIRST_CHUNK_TREE_OBJECTID, CHUNK_ITEM_KEY, log), process_func)
cur_logical = next(iter(self._chunk_map)) # first item
if cur_logical > log:
raise Exception(f'Cannot translate address {log:#x}')
for logical, cmi in self._chunk_map.items():
if logical > log:
break
cur_logical = logical
if cur_logical + self._chunk_map[cur_logical].length < log:
raise Exception(f'Cannot translate address {log:#x}')
print('address translation: {:#x} -> {:#x}'.format(log, self._chunk_map[cur_logical].physical + log - cur_logical))
return self.part_offset + self._chunk_map[cur_logical].physical + log - cur_logical
def search_tree(self, level, root_offset, key, process_node_func = None):
for lvl in range(level, 0, -1):
# inner node
root_offset = self.logical_to_physical(root_offset)
self.fd.seek(root_offset)
header = NodeHeader._make(_node_header_struct.unpack(self.fd.read(_node_header_struct.size)))
if header.level != lvl:
raise Exception('Invalid inner node level')
found, itemnr = key_bin_search(
self.fd,
root_offset + _node_header_struct.size,
InnerKey.sstruct.size,
key,
0,
header.items_num
)
# TODO: better understand this
if not found and itemnr > 0:
itemnr -= 1
self.fd.seek(root_offset + _node_header_struct.size + itemnr * InnerKey.sstruct.size)
k = InnerKey(self.fd.read(InnerKey.sstruct.size))
root_offset = k.block_num
else:
# we are in leaf node
root_offset = self.logical_to_physical(root_offset)
self.fd.seek(root_offset)
header = NodeHeader._make(_node_header_struct.unpack(self.fd.read(_node_header_struct.size)))
if header.level != 0:
raise Exception('Invalid leaf level')
if process_node_func:
process_node_func(header, root_offset + _node_header_struct.size)
return root_offset
found, itemnr = key_bin_search(
self.fd,
root_offset + _node_header_struct.size,
LeafKey.sstruct.size,
key,
0,
header.items_num
)
self.fd.seek(root_offset + _node_header_struct.size + itemnr * LeafKey.sstruct.size)
k = LeafKey(self.fd.read(LeafKey.sstruct.size))
self.fd.seek(root_offset + _node_header_struct.size + k.data_offset)
if k.type in _key_type_class_map:
return k, _key_type_class_map[k.type](k, self.fd.read(k.data_size))
else:
return k, False
def print_node(self, header, offset):
print(header)
root_paddr = self.logical_to_physical(header.node_addr)
key_size = InnerKey.sstruct.size if header.level > 0 else LeafKey.sstruct.size
key_struct = InnerKey if header.level > 0 else LeafKey
for i in range(header.items_num):
self.fd.seek(root_paddr + _node_header_struct.size + i * key_size)
k = key_struct(self.fd.read(key_size))
print(k)
self.fd.seek(root_paddr + _node_header_struct.size + k.data_offset)
if k.type in _key_type_class_map and header.level == 0:
item = _key_type_class_map[k.type](k, self.fd.read(k.data_size))
print(item)
if k.type == DIR_ITEM_KEY:
for it in item:
print(it)
print('============================')
def print_chunk_map(self):
print('=== chunk map ===')
for logical, cmi in self._chunk_map.items():
print(f'{cmi.logical:#x}..{cmi.logical+cmi.length:#x} -> {cmi.physical:#x}..{cmi.physical+cmi.length:#x}')
print('=================')
def _insert_chunk(self, chunk):
if not chunk.logical in self._chunk_map:
cm = dict(self._chunk_map)
cm[chunk.logical] = chunk
self._chunk_map = OrderedDict(sorted(cm.items()))
_key_type_class_map = {
INODE_ITEM_KEY: InodeItem,
INODE_REF_KEY: InodeRef,
DIR_ITEM_KEY: DirItemList,
DIR_INDEX_KEY: DirIndex,
EXTENT_DATA_KEY: FileExtentItem,
ROOT_ITEM_KEY: RootItem,
EXTENT_ITEM_KEY: ExtentItem,
CHUNK_ITEM_KEY: Chunk,
}
Reference in a new issue View git blame Copy permalink