mirror of
https://github.com/reactos/reactos.git
synced 2024-08-20 10:23:22 +00:00
1293 lines
37 KiB
Python
1293 lines
37 KiB
Python
# PROJECT: Python tools for traversing BTRFS structures
|
|
# LICENSE: GPL-2.0+ (https://spdx.org/licenses/GPL-2.0+)
|
|
# PURPOSE: Classes and structures for BTRFS on-disk layout
|
|
# COPYRIGHT: Copyright 2018 Victor Perevertkin (victor@perevertkin.ru)
|
|
|
|
# some code was taken from https://github.com/knorrie/python-btrfs
|
|
|
|
from btrfs_constants import *
|
|
import struct
|
|
from collections import namedtuple, OrderedDict
|
|
import collections.abc
|
|
import copy
|
|
import datetime
|
|
import os
|
|
import uuid
|
|
import crc32c
|
|
|
|
ULLONG_MAX = (1 << 64) - 1
|
|
ULONG_MAX = (1 << 32) - 1
|
|
|
|
|
|
def ULL(n):
|
|
return n & ULLONG_MAX
|
|
|
|
|
|
ROOT_TREE_OBJECTID = 1
|
|
EXTENT_TREE_OBJECTID = 2
|
|
CHUNK_TREE_OBJECTID = 3
|
|
DEV_TREE_OBJECTID = 4
|
|
FS_TREE_OBJECTID = 5
|
|
ROOT_TREE_DIR_OBJECTID = 6
|
|
CSUM_TREE_OBJECTID = 7
|
|
QUOTA_TREE_OBJECTID = 8
|
|
UUID_TREE_OBJECTID = 9
|
|
FREE_SPACE_TREE_OBJECTID = 10
|
|
|
|
DEV_STATS_OBJECTID = 0
|
|
BALANCE_OBJECTID = ULL(-4)
|
|
ORPHAN_OBJECTID = ULL(-5)
|
|
TREE_LOG_OBJECTID = ULL(-6)
|
|
TREE_LOG_FIXUP_OBJECTID = ULL(-7)
|
|
TREE_RELOC_OBJECTID = ULL(-8)
|
|
DATA_RELOC_TREE_OBJECTID = ULL(-9)
|
|
EXTENT_CSUM_OBJECTID = ULL(-10)
|
|
FREE_SPACE_OBJECTID = ULL(-11)
|
|
FREE_INO_OBJECTID = ULL(-12)
|
|
MULTIPLE_OBJECTIDS = ULL(-255)
|
|
|
|
FIRST_FREE_OBJECTID = 256
|
|
LAST_FREE_OBJECTID = ULL(-256)
|
|
FIRST_CHUNK_TREE_OBJECTID = 256
|
|
|
|
DEV_ITEMS_OBJECTID = 1
|
|
|
|
BTRFS_SYSTEM_CHUNK_ARRAY_SIZE = 2048
|
|
|
|
|
|
INODE_ITEM_KEY = 1
|
|
INODE_REF_KEY = 12
|
|
INODE_EXTREF_KEY = 13
|
|
XATTR_ITEM_KEY = 24
|
|
ORPHAN_ITEM_KEY = 48
|
|
DIR_LOG_ITEM_KEY = 60
|
|
DIR_LOG_INDEX_KEY = 72
|
|
DIR_ITEM_KEY = 84
|
|
DIR_INDEX_KEY = 96
|
|
EXTENT_DATA_KEY = 108
|
|
EXTENT_CSUM_KEY = 128
|
|
ROOT_ITEM_KEY = 132
|
|
ROOT_BACKREF_KEY = 144
|
|
ROOT_REF_KEY = 156
|
|
EXTENT_ITEM_KEY = 168
|
|
METADATA_ITEM_KEY = 169
|
|
TREE_BLOCK_REF_KEY = 176
|
|
EXTENT_DATA_REF_KEY = 178
|
|
SHARED_BLOCK_REF_KEY = 182
|
|
SHARED_DATA_REF_KEY = 184
|
|
BLOCK_GROUP_ITEM_KEY = 192
|
|
FREE_SPACE_INFO_KEY = 198
|
|
FREE_SPACE_EXTENT_KEY = 199
|
|
FREE_SPACE_BITMAP_KEY = 200
|
|
DEV_EXTENT_KEY = 204
|
|
DEV_ITEM_KEY = 216
|
|
CHUNK_ITEM_KEY = 228
|
|
QGROUP_STATUS_KEY = 240
|
|
QGROUP_INFO_KEY = 242
|
|
QGROUP_LIMIT_KEY = 244
|
|
QGROUP_RELATION_KEY = 246
|
|
BALANCE_ITEM_KEY = 248
|
|
DEV_STATS_KEY = 249
|
|
DEV_REPLACE_KEY = 250
|
|
UUID_KEY_SUBVOL = 251
|
|
UUID_KEY_RECEIVED_SUBVOL = 252
|
|
STRING_ITEM_KEY = 253
|
|
|
|
BLOCK_GROUP_SINGLE = 0
|
|
BLOCK_GROUP_DATA = 1 << 0
|
|
BLOCK_GROUP_SYSTEM = 1 << 1
|
|
BLOCK_GROUP_METADATA = 1 << 2
|
|
BLOCK_GROUP_RAID0 = 1 << 3
|
|
BLOCK_GROUP_RAID1 = 1 << 4
|
|
BLOCK_GROUP_DUP = 1 << 5
|
|
BLOCK_GROUP_RAID10 = 1 << 6
|
|
BLOCK_GROUP_RAID5 = 1 << 7
|
|
BLOCK_GROUP_RAID6 = 1 << 8
|
|
|
|
BLOCK_GROUP_TYPE_MASK = (
|
|
BLOCK_GROUP_DATA |
|
|
BLOCK_GROUP_SYSTEM |
|
|
BLOCK_GROUP_METADATA
|
|
)
|
|
|
|
BLOCK_GROUP_PROFILE_MASK = (
|
|
BLOCK_GROUP_RAID0 |
|
|
BLOCK_GROUP_RAID1 |
|
|
BLOCK_GROUP_RAID5 |
|
|
BLOCK_GROUP_RAID6 |
|
|
BLOCK_GROUP_DUP |
|
|
BLOCK_GROUP_RAID10
|
|
)
|
|
|
|
AVAIL_ALLOC_BIT_SINGLE = 1 << 48 # used in balance_args
|
|
SPACE_INFO_GLOBAL_RSV = 1 << 49
|
|
|
|
|
|
_block_group_flags_str_map = {
|
|
BLOCK_GROUP_DATA: 'DATA',
|
|
BLOCK_GROUP_METADATA: 'METADATA',
|
|
BLOCK_GROUP_SYSTEM: 'SYSTEM',
|
|
BLOCK_GROUP_RAID0: 'RAID0',
|
|
BLOCK_GROUP_RAID1: 'RAID1',
|
|
BLOCK_GROUP_DUP: 'DUP',
|
|
BLOCK_GROUP_RAID10: 'RAID10',
|
|
BLOCK_GROUP_RAID5: 'RAID5',
|
|
BLOCK_GROUP_RAID6: 'RAID6',
|
|
}
|
|
|
|
_balance_args_profiles_str_map = {
|
|
BLOCK_GROUP_RAID0: 'RAID0',
|
|
BLOCK_GROUP_RAID1: 'RAID1',
|
|
BLOCK_GROUP_DUP: 'DUP',
|
|
BLOCK_GROUP_RAID10: 'RAID10',
|
|
BLOCK_GROUP_RAID5: 'RAID5',
|
|
BLOCK_GROUP_RAID6: 'RAID6',
|
|
AVAIL_ALLOC_BIT_SINGLE: 'SINGLE',
|
|
}
|
|
|
|
QGROUP_LEVEL_SHIFT = 48
|
|
|
|
EXTENT_FLAG_DATA = 1 << 0
|
|
EXTENT_FLAG_TREE_BLOCK = 1 << 1
|
|
BLOCK_FLAG_FULL_BACKREF = 1 << 8
|
|
|
|
_extent_flags_str_map = {
|
|
EXTENT_FLAG_DATA: 'DATA',
|
|
EXTENT_FLAG_TREE_BLOCK: 'TREE_BLOCK',
|
|
BLOCK_FLAG_FULL_BACKREF: 'FULL_BACKREF',
|
|
}
|
|
|
|
INODE_NODATASUM = 1 << 0
|
|
INODE_NODATACOW = 1 << 1
|
|
INODE_READONLY = 1 << 2
|
|
INODE_NOCOMPRESS = 1 << 3
|
|
INODE_PREALLOC = 1 << 4
|
|
INODE_SYNC = 1 << 5
|
|
INODE_IMMUTABLE = 1 << 6
|
|
INODE_APPEND = 1 << 7
|
|
INODE_NODUMP = 1 << 8
|
|
INODE_NOATIME = 1 << 9
|
|
INODE_DIRSYNC = 1 << 10
|
|
INODE_COMPRESS = 1 << 11
|
|
|
|
_inode_flags_str_map = {
|
|
INODE_NODATASUM: 'NODATASUM',
|
|
INODE_READONLY: 'READONLY',
|
|
INODE_NOCOMPRESS: 'NOCOMPRESS',
|
|
INODE_PREALLOC: 'PREALLOC',
|
|
INODE_SYNC: 'SYNC',
|
|
INODE_IMMUTABLE: 'IMMUTABLE',
|
|
INODE_APPEND: 'APPEND',
|
|
INODE_NODUMP: 'NODUMP',
|
|
INODE_NOATIME: 'NOATIME',
|
|
INODE_DIRSYNC: 'DIRSYNC',
|
|
INODE_COMPRESS: 'COMPRESS',
|
|
}
|
|
|
|
ROOT_SUBVOL_RDONLY = 1 << 0
|
|
|
|
_root_flags_str_map = {
|
|
ROOT_SUBVOL_RDONLY: 'RDONLY',
|
|
}
|
|
|
|
FT_UNKNOWN = 0
|
|
FT_REG_FILE = 1
|
|
FT_DIR = 2
|
|
FT_CHRDEV = 3
|
|
FT_BLKDEV = 4
|
|
FT_FIFO = 5
|
|
FT_SOCK = 6
|
|
FT_SYMLINK = 7
|
|
FT_XATTR = 8
|
|
FT_MAX = 9
|
|
|
|
_dir_item_type_str_map = {
|
|
FT_UNKNOWN: 'UNKNOWN',
|
|
FT_REG_FILE: 'FILE',
|
|
FT_DIR: 'DIR',
|
|
FT_CHRDEV: 'CHRDEV',
|
|
FT_BLKDEV: 'BLKDEV',
|
|
FT_FIFO: 'FIFO',
|
|
FT_SOCK: 'SOCK',
|
|
FT_SYMLINK: 'SYMLINK',
|
|
FT_XATTR: 'XATTR',
|
|
}
|
|
|
|
COMPRESS_NONE = 0
|
|
COMPRESS_ZLIB = 1
|
|
COMPRESS_LZO = 2
|
|
COMPRESS_ZSTD = 3
|
|
|
|
_compress_type_str_map = {
|
|
COMPRESS_NONE: 'none',
|
|
COMPRESS_ZLIB: 'zlib',
|
|
COMPRESS_LZO: 'lzo',
|
|
COMPRESS_ZSTD: 'zstd',
|
|
}
|
|
|
|
FILE_EXTENT_INLINE = 0
|
|
FILE_EXTENT_REG = 1
|
|
FILE_EXTENT_PREALLOC = 2
|
|
|
|
_file_extent_type_str_map = {
|
|
FILE_EXTENT_INLINE: 'inline',
|
|
FILE_EXTENT_REG: 'regular',
|
|
FILE_EXTENT_PREALLOC: 'prealloc',
|
|
}
|
|
|
|
|
|
def qgroup_level(objectid):
|
|
return objectid >> QGROUP_LEVEL_SHIFT
|
|
|
|
|
|
def qgroup_subvid(objectid):
|
|
return objectid & ((1 << QGROUP_LEVEL_SHIFT) - 1)
|
|
|
|
|
|
_key_objectid_str_map = {
|
|
ROOT_TREE_OBJECTID: 'ROOT_TREE',
|
|
EXTENT_TREE_OBJECTID: 'EXTENT_TREE',
|
|
CHUNK_TREE_OBJECTID: 'CHUNK_TREE',
|
|
DEV_TREE_OBJECTID: 'DEV_TREE',
|
|
FS_TREE_OBJECTID: 'FS_TREE',
|
|
ROOT_TREE_DIR_OBJECTID: 'ROOT_TREE_DIR',
|
|
CSUM_TREE_OBJECTID: 'CSUM_TREE',
|
|
QUOTA_TREE_OBJECTID: 'QUOTA_TREE',
|
|
UUID_TREE_OBJECTID: 'UUID_TREE',
|
|
FREE_SPACE_TREE_OBJECTID: 'FREE_SPACE_TREE',
|
|
BALANCE_OBJECTID: 'BALANCE',
|
|
ORPHAN_OBJECTID: 'ORPHAN',
|
|
TREE_LOG_OBJECTID: 'TREE_LOG',
|
|
TREE_LOG_FIXUP_OBJECTID: 'TREE_LOG_FIXUP',
|
|
TREE_RELOC_OBJECTID: 'TREE_RELOC',
|
|
DATA_RELOC_TREE_OBJECTID: 'DATA_RELOC_TREE',
|
|
EXTENT_CSUM_OBJECTID: 'EXTENT_CSUM',
|
|
FREE_SPACE_OBJECTID: 'FREE_SPACE',
|
|
FREE_INO_OBJECTID: 'FREE_INO',
|
|
MULTIPLE_OBJECTIDS: 'MULTIPLE',
|
|
}
|
|
|
|
|
|
def key_objectid_str(objectid, _type):
|
|
if _type == DEV_EXTENT_KEY:
|
|
return str(objectid)
|
|
if _type == QGROUP_RELATION_KEY:
|
|
return "{}/{}".format(qgroup_level(objectid), qgroup_subvid(objectid))
|
|
if _type == UUID_KEY_SUBVOL or _type == UUID_KEY_RECEIVED_SUBVOL:
|
|
return "0x{:0>16x}".format(objectid)
|
|
|
|
if objectid == ROOT_TREE_OBJECTID and _type == DEV_ITEM_KEY:
|
|
return 'DEV_ITEMS'
|
|
if objectid == DEV_STATS_OBJECTID and _type == DEV_STATS_KEY:
|
|
return 'DEV_STATS'
|
|
if objectid == FIRST_CHUNK_TREE_OBJECTID and _type == CHUNK_ITEM_KEY:
|
|
return 'FIRST_CHUNK_TREE'
|
|
if objectid == ULLONG_MAX:
|
|
return '-1'
|
|
|
|
return _key_objectid_str_map.get(objectid, str(objectid))
|
|
|
|
|
|
_key_type_str_map = {
|
|
INODE_ITEM_KEY: 'INODE_ITEM',
|
|
INODE_REF_KEY: 'INODE_REF',
|
|
INODE_EXTREF_KEY: 'INODE_EXTREF',
|
|
XATTR_ITEM_KEY: 'XATTR_ITEM',
|
|
ORPHAN_ITEM_KEY: 'ORPHAN_ITEM',
|
|
DIR_LOG_ITEM_KEY: 'DIR_LOG_ITEM',
|
|
DIR_LOG_INDEX_KEY: 'DIR_LOG_INDEX',
|
|
DIR_ITEM_KEY: 'DIR_ITEM',
|
|
DIR_INDEX_KEY: 'DIR_INDEX',
|
|
EXTENT_DATA_KEY: 'EXTENT_DATA',
|
|
EXTENT_CSUM_KEY: 'EXTENT_CSUM',
|
|
ROOT_ITEM_KEY: 'ROOT_ITEM',
|
|
ROOT_BACKREF_KEY: 'ROOT_BACKREF',
|
|
ROOT_REF_KEY: 'ROOT_REF',
|
|
EXTENT_ITEM_KEY: 'EXTENT_ITEM',
|
|
METADATA_ITEM_KEY: 'METADATA_ITEM',
|
|
TREE_BLOCK_REF_KEY: 'TREE_BLOCK_REF',
|
|
EXTENT_DATA_REF_KEY: 'EXTENT_DATA_REF',
|
|
SHARED_BLOCK_REF_KEY: 'SHARED_BLOCK_REF',
|
|
SHARED_DATA_REF_KEY: 'SHARED_DATA_REF',
|
|
BLOCK_GROUP_ITEM_KEY: 'BLOCK_GROUP_ITEM',
|
|
FREE_SPACE_INFO_KEY: 'FREE_SPACE_INFO',
|
|
FREE_SPACE_EXTENT_KEY: 'FREE_SPACE_EXTENT',
|
|
FREE_SPACE_BITMAP_KEY: 'FREE_SPACE_BITMAP',
|
|
DEV_EXTENT_KEY: 'DEV_EXTENT',
|
|
DEV_ITEM_KEY: 'DEV_ITEM',
|
|
CHUNK_ITEM_KEY: 'CHUNK_ITEM',
|
|
QGROUP_STATUS_KEY: 'QGROUP_STATUS',
|
|
QGROUP_INFO_KEY: 'QGROUP_INFO',
|
|
QGROUP_LIMIT_KEY: 'QGROUP_LIMIT',
|
|
QGROUP_RELATION_KEY: 'QGROUP_RELATION',
|
|
BALANCE_ITEM_KEY: 'BALANCE_ITEM',
|
|
DEV_STATS_KEY: 'DEV_STATS',
|
|
DEV_REPLACE_KEY: 'DEV_REPLACE',
|
|
UUID_KEY_SUBVOL: 'UUID_SUBVOL',
|
|
UUID_KEY_RECEIVED_SUBVOL: 'RECEIVED_SUBVOL',
|
|
STRING_ITEM_KEY: 'STRING_ITEM',
|
|
}
|
|
|
|
# === Helper functions
|
|
|
|
def key_type_str(_type):
|
|
return _key_type_str_map.get(_type, str(_type))
|
|
|
|
|
|
def key_offset_str(offset, _type):
|
|
if _type == QGROUP_RELATION_KEY or _type == QGROUP_INFO_KEY or _type == QGROUP_LIMIT_KEY:
|
|
return "{}/{}".format(qgroup_level(offset), qgroup_subvid(offset))
|
|
if _type == UUID_KEY_SUBVOL or _type == UUID_KEY_RECEIVED_SUBVOL:
|
|
return "0x{:0>16x}".format(offset)
|
|
if _type == ROOT_ITEM_KEY:
|
|
return _key_objectid_str_map.get(offset, str(offset))
|
|
if offset == ULLONG_MAX:
|
|
return '-1'
|
|
|
|
return str(offset)
|
|
|
|
|
|
def flags_str(flags, flags_str_map):
|
|
ret = []
|
|
for flag in sorted(flags_str_map.keys()):
|
|
if flags & flag:
|
|
ret.append(flags_str_map[flag])
|
|
if len(ret) == 0:
|
|
ret.append("none")
|
|
return '|'.join(ret)
|
|
|
|
|
|
def embedded_text_for_str(text):
|
|
try:
|
|
return "utf-8 {}".format(text.decode('utf-8'))
|
|
except UnicodeDecodeError:
|
|
return "raw {}".format(repr(text))
|
|
|
|
|
|
# === Basic structures
|
|
|
|
|
|
class TimeSpec(object):
|
|
sstruct = struct.Struct('<QL')
|
|
|
|
@staticmethod
|
|
def from_values(sec, nsec):
|
|
t = TimeSpec.__new__(TimeSpec)
|
|
t.sec = sec
|
|
t.nsec = nsec
|
|
return t
|
|
|
|
def __init__(self, data):
|
|
self.sec, self.nsec = TimeSpec.sstruct.unpack_from(data)
|
|
|
|
@property
|
|
def iso8601(self):
|
|
return datetime.datetime.utcfromtimestamp(
|
|
float("{self.sec}.{self.nsec}".format(self=self))
|
|
).isoformat()
|
|
|
|
def __str__(self):
|
|
return "{self.sec}.{self.nsec} ({self.iso8601})".format(self=self)
|
|
|
|
|
|
class Key(object):
|
|
def __init__(self, objectid, _type, offset):
|
|
self._objectid = objectid
|
|
self._type = _type
|
|
self._offset = offset
|
|
self._pack()
|
|
|
|
@property
|
|
def objectid(self):
|
|
return self._objectid
|
|
|
|
@objectid.setter
|
|
def objectid(self, _objectid):
|
|
self._objectid = _objectid
|
|
self._pack()
|
|
|
|
@property
|
|
def type(self):
|
|
return self._type
|
|
|
|
@type.setter
|
|
def type(self, _type):
|
|
self._type = _type
|
|
self._pack()
|
|
|
|
@property
|
|
def offset(self):
|
|
return self._offset
|
|
|
|
@offset.setter
|
|
def offset(self, _offset):
|
|
self._offset = _offset
|
|
self._pack()
|
|
|
|
@property
|
|
def key(self):
|
|
return self._key
|
|
|
|
@key.setter
|
|
def key(self, _key):
|
|
self._key = _key
|
|
self._unpack()
|
|
|
|
def _pack(self):
|
|
self._key = (self.objectid << 72) + (self._type << 64) + self.offset
|
|
|
|
def _unpack(self):
|
|
self._objectid = self._key >> 72
|
|
self._type = (self._key & ((1 << 72) - 1)) >> 64
|
|
self._offset = (self._key & ((1 << 64) - 1))
|
|
|
|
def __lt__(self, other):
|
|
if isinstance(other, Key):
|
|
return self._key < other._key
|
|
return self._key < other
|
|
|
|
def __le__(self, other):
|
|
if isinstance(other, Key):
|
|
return self._key <= other._key
|
|
return self._key <= other
|
|
|
|
def __eq__(self, other):
|
|
if isinstance(other, Key):
|
|
return self._key == other._key
|
|
return self._key == other
|
|
|
|
def __ge__(self, other):
|
|
if isinstance(other, Key):
|
|
return self._key >= other._key
|
|
return self._key >= other
|
|
|
|
def __gt__(self, other):
|
|
if isinstance(other, Key):
|
|
return self._key > other._key
|
|
return self._key > other
|
|
|
|
def __str__(self):
|
|
return "({} {} {})".format(
|
|
key_objectid_str(self._objectid, self._type),
|
|
key_type_str(self._type),
|
|
key_offset_str(self._offset, self._type),
|
|
)
|
|
|
|
def __add__(self, amount):
|
|
new_key = copy.copy(self)
|
|
new_key.key += amount
|
|
return new_key
|
|
|
|
def __sub__(self, amount):
|
|
new_key = copy.copy(self)
|
|
new_key.key -= amount
|
|
return new_key
|
|
|
|
class DiskKey(Key):
|
|
sstruct = struct.Struct('<QBQ')
|
|
|
|
def __init__(self, data):
|
|
super(DiskKey, self).__init__(*DiskKey.sstruct.unpack_from(data))
|
|
|
|
class InnerKey(Key):
|
|
sstruct = struct.Struct('<QBQQQ')
|
|
|
|
def __init__(self, data):
|
|
unpacked_data = InnerKey.sstruct.unpack_from(data)
|
|
super().__init__(*unpacked_data[:3])
|
|
self.block_num = unpacked_data[3]
|
|
self.generation = unpacked_data[4]
|
|
|
|
def __str__(self):
|
|
return "(inner_key {} {} {} block_num {}, generation {})".format(
|
|
key_objectid_str(self._objectid, self._type),
|
|
key_type_str(self._type),
|
|
key_offset_str(self._offset, self._type),
|
|
self.block_num,
|
|
self.generation,
|
|
)
|
|
|
|
class LeafKey(Key):
|
|
sstruct = struct.Struct('<QBQLL')
|
|
|
|
def __init__(self, data):
|
|
unpacked_data = LeafKey.sstruct.unpack_from(data)
|
|
super().__init__(*unpacked_data[:3])
|
|
self.data_offset = unpacked_data[3]
|
|
self.data_size = unpacked_data[4]
|
|
|
|
def __str__(self):
|
|
return "(leaf_key {} {} {} data_offset {:#x} data_size {})".format(
|
|
key_objectid_str(self._objectid, self._type),
|
|
key_type_str(self._type),
|
|
key_offset_str(self._offset, self._type),
|
|
self.data_offset,
|
|
self.data_size,
|
|
)
|
|
|
|
class ItemData(object):
|
|
def __init__(self, key):
|
|
self.key = key
|
|
|
|
def setattr_from_key(self, objectid_attr=None, type_attr=None, offset_attr=None):
|
|
if objectid_attr is not None:
|
|
setattr(self, objectid_attr, self.key.objectid)
|
|
if type_attr is not None:
|
|
setattr(self, type_attr, self.key.type)
|
|
if offset_attr is not None:
|
|
setattr(self, offset_attr, self.key.offset)
|
|
self._key_attrs = objectid_attr, type_attr, offset_attr
|
|
|
|
@property
|
|
def key_attrs(self):
|
|
try:
|
|
return self._key_attrs
|
|
except AttributeError:
|
|
return None, None, None
|
|
|
|
def __lt__(self, other):
|
|
return self.key < other.key
|
|
|
|
|
|
superblock = struct.Struct('<32x16s2Q8s9Q5L4QH2B611x2048s')
|
|
# NOTE: the structure is not complete
|
|
# FS UUID
|
|
# Physical block address
|
|
# Flags
|
|
# Signature (_BHRfS_M)
|
|
# generation
|
|
# Log. address of root of tree roots
|
|
# Log. address of chunk tree root
|
|
# Log. address of log tree root
|
|
# log_root_transid
|
|
# total_bytes
|
|
# bytes_used
|
|
# root_dir_objectid (usually 6)
|
|
# num_devices
|
|
# sectorsize
|
|
# nodesize
|
|
# __unused_leafsize
|
|
# stripesize
|
|
# sys_chunk_array_size
|
|
# chunk_root_generation
|
|
# compat_flags
|
|
# compat_ro_flags
|
|
# incompat_flags
|
|
# csum_type
|
|
# root_level 23
|
|
# chunk_root_level 24
|
|
# ---
|
|
# sys_chunk_array
|
|
|
|
|
|
_node_header_struct = struct.Struct('<32x16sQQ16sQQLB')
|
|
NodeHeader = namedtuple('NodeHeader', 'FS_UUID node_addr flags chunk_tree_uuid generation tree_id items_num level')
|
|
|
|
# === Items
|
|
|
|
class InodeItem(ItemData):
|
|
_inode_item = [
|
|
struct.Struct('<5Q4L3Q32x'),
|
|
TimeSpec.sstruct,
|
|
TimeSpec.sstruct,
|
|
TimeSpec.sstruct,
|
|
TimeSpec.sstruct,
|
|
]
|
|
sstruct = struct.Struct('<' + ''.join([s.format[1:].decode() for s in _inode_item]))
|
|
|
|
def __init__(self, key, data):
|
|
super().__init__(key)
|
|
self.generation, self.transid, self.size, self.nbytes, self.block_group, \
|
|
self.nlink, self.uid, self.gid, self.mode, self.rdev, self.flags, self.sequence = \
|
|
InodeItem._inode_item[0].unpack_from(data)
|
|
pos = InodeItem._inode_item[0].size
|
|
next_pos = pos + TimeSpec.sstruct.size
|
|
self.atime = TimeSpec(data[pos:next_pos])
|
|
pos, next_pos = next_pos, next_pos + TimeSpec.sstruct.size
|
|
self.ctime = TimeSpec(data[pos:next_pos])
|
|
pos, next_pos = next_pos, next_pos + TimeSpec.sstruct.size
|
|
self.mtime = TimeSpec(data[pos:next_pos])
|
|
pos, next_pos = next_pos, next_pos + TimeSpec.sstruct.size
|
|
self.otime = TimeSpec(data[pos:next_pos])
|
|
|
|
@property
|
|
def flags_str(self):
|
|
return flags_str(self.flags, _inode_flags_str_map)
|
|
|
|
def __str__(self):
|
|
return "inode generation {self.generation} transid {self.transid} size {self.size} " \
|
|
"nbytes {self.nbytes} block_group {self.block_group} mode {self.mode:05o} " \
|
|
"nlink {self.nlink} uid {self.uid} gid {self.gid} rdev {self.rdev} " \
|
|
"flags {self.flags:#x}({self.flags_str})".format(self=self)
|
|
|
|
|
|
class RootItem(ItemData):
|
|
_root_item = [
|
|
InodeItem.sstruct,
|
|
struct.Struct('<7QL'),
|
|
DiskKey.sstruct,
|
|
struct.Struct('<BBQ16s16s16s4Q'),
|
|
TimeSpec.sstruct,
|
|
TimeSpec.sstruct,
|
|
TimeSpec.sstruct,
|
|
TimeSpec.sstruct,
|
|
]
|
|
sstruct = struct.Struct('<' + ''.join([s.format[1:].decode() for s in _root_item]))
|
|
|
|
def __init__(self, key, data):
|
|
super().__init__(key)
|
|
self.inode = InodeItem(None, data[:InodeItem.sstruct.size])
|
|
pos = InodeItem.sstruct.size
|
|
self.generation, self.dirid, self.bytenr, self.byte_limit, self.bytes_used, \
|
|
self.last_snapshot, self.flags, self.refs = \
|
|
RootItem._root_item[1].unpack_from(data, pos)
|
|
pos += RootItem._root_item[1].size
|
|
self.drop_progress = DiskKey(data[pos:pos+DiskKey.sstruct.size])
|
|
pos += DiskKey.sstruct.size
|
|
self.drop_level, self.level, self.generation_v2, uuid_bytes, parent_uuid_bytes, \
|
|
received_uuid_bytes, self.ctransid, self.otransid, self.stransid, self.rtransid = \
|
|
RootItem._root_item[3].unpack_from(data, pos)
|
|
self.uuid = uuid.UUID(bytes=uuid_bytes)
|
|
self.parent_uuid = uuid.UUID(bytes=parent_uuid_bytes)
|
|
self.received_uuid = uuid.UUID(bytes=received_uuid_bytes)
|
|
pos += RootItem._root_item[3].size
|
|
next_pos = pos + TimeSpec.sstruct.size
|
|
self.ctime = TimeSpec(data[pos:next_pos])
|
|
pos, next_pos = next_pos, next_pos + TimeSpec.sstruct.size
|
|
self.otime = TimeSpec(data[pos:next_pos])
|
|
pos, next_pos = next_pos, next_pos + TimeSpec.sstruct.size
|
|
self.stime = TimeSpec(data[pos:next_pos])
|
|
pos, next_pos = next_pos, next_pos + TimeSpec.sstruct.size
|
|
self.rtime = TimeSpec(data[pos:next_pos])
|
|
|
|
@property
|
|
def flags_str(self):
|
|
return flags_str(self.flags, _root_flags_str_map)
|
|
|
|
def __str__(self):
|
|
return "root {self.key.objectid} uuid {self.uuid} " \
|
|
"generation {self.generation} last_snapshot {self.last_snapshot} " \
|
|
"bytenr {self.bytenr:#x} level {self.level} " \
|
|
"flags {self.flags:#x}({self.flags_str})".format(self=self)
|
|
|
|
|
|
class Chunk(ItemData):
|
|
sstruct = struct.Struct('<4Q3L2H')
|
|
|
|
def __init__(self, key, data):
|
|
super().__init__(key)
|
|
self.setattr_from_key(offset_attr='vaddr')
|
|
self.length, self.owner, self.stripe_len, self.type, self.io_align, \
|
|
self.io_width, self.sector_size, self.num_stripes, self.sub_stripes = \
|
|
Chunk.sstruct.unpack_from(data)
|
|
self.stripes = []
|
|
pos = Chunk.sstruct.size
|
|
for i in range(self.num_stripes):
|
|
next_pos = pos + Stripe.sstruct.size
|
|
self.stripes.append(Stripe(data[pos:next_pos]))
|
|
pos = next_pos
|
|
|
|
@property
|
|
def size(self):
|
|
return Chunk.sstruct.size + self.num_stripes * Stripe.sstruct.size
|
|
|
|
@property
|
|
def type_str(self):
|
|
return flags_str(self.type, _block_group_flags_str_map)
|
|
|
|
def __str__(self):
|
|
return "chunk vaddr {self.vaddr:#x} type {self.type_str} length {self.length} " \
|
|
"num_stripes {self.num_stripes}".format(self=self)
|
|
|
|
|
|
class Stripe(object):
|
|
sstruct = struct.Struct('<2Q16s')
|
|
|
|
def __init__(self, data):
|
|
self.devid, self.offset, uuid_bytes = Stripe.sstruct.unpack(data)
|
|
self.uuid = uuid.UUID(bytes=uuid_bytes)
|
|
|
|
def __str__(self):
|
|
return "stripe devid {self.devid} offset {self.offset:#x}".format(self=self)
|
|
|
|
|
|
class InodeRefList(ItemData, collections.abc.MutableSequence):
|
|
def __init__(self, header, data):
|
|
super().__init__(header)
|
|
self._list = []
|
|
pos = 0
|
|
while pos < header.len:
|
|
inode_ref = InodeRef(data, pos)
|
|
self._list.append(inode_ref)
|
|
pos += len(inode_ref)
|
|
|
|
def __getitem__(self, index):
|
|
return self._list[index]
|
|
|
|
def __setitem__(self, index, value):
|
|
self._list[index] = value
|
|
|
|
def __delitem__(self, index):
|
|
del self._list[index]
|
|
|
|
def __len__(self):
|
|
return len(self._list)
|
|
|
|
def insert(self, index, value):
|
|
self._list.insert(index, value)
|
|
|
|
def __str__(self):
|
|
return "inode ref list size {}".format(len(self))
|
|
|
|
|
|
class InodeRef(ItemData):
|
|
sstruct = struct.Struct('<QH')
|
|
|
|
def __init__(self, key, data):
|
|
super().__init__(key)
|
|
self.index, self.name_len = InodeRef.sstruct.unpack_from(data)
|
|
self.name, = struct.Struct('<{}s'.format(self.name_len)).unpack_from(data, InodeRef.sstruct.size)
|
|
self._len = InodeRef.sstruct.size + self.name_len
|
|
|
|
@property
|
|
def name_str(self):
|
|
return embedded_text_for_str(self.name)
|
|
|
|
def __len__(self):
|
|
return self._len
|
|
|
|
def __str__(self):
|
|
return "inode ref index {self.index} name {self.name_str}".format(self=self)
|
|
|
|
|
|
class DirItemList(ItemData, collections.abc.MutableSequence):
|
|
def __init__(self, key, data):
|
|
super().__init__(key)
|
|
self._list = []
|
|
pos = 0
|
|
while pos < key.data_size:
|
|
cls = {DIR_ITEM_KEY: DirItem, XATTR_ITEM_KEY: XAttrItem}
|
|
dir_item = cls[self.key.type](data, pos)
|
|
self._list.append(dir_item)
|
|
pos += len(dir_item)
|
|
|
|
def __getitem__(self, index):
|
|
return self._list[index]
|
|
|
|
def __setitem__(self, index, value):
|
|
self._list[index] = value
|
|
|
|
def __delitem__(self, index):
|
|
del self._list[index]
|
|
|
|
def __len__(self):
|
|
return len(self._list)
|
|
|
|
def insert(self, index, value):
|
|
self._list.insert(index, value)
|
|
|
|
def __str__(self):
|
|
return "dir item list hash {self.key.offset} size {}".format(len(self), self=self)
|
|
|
|
|
|
class XAttrItemList(DirItemList):
|
|
def __str__(self):
|
|
return "xattr item list hash {self.key.offset} size {}".format(len(self), self=self)
|
|
|
|
|
|
class DirItem(object):
|
|
_dir_item = [
|
|
DiskKey.sstruct,
|
|
struct.Struct('<QHHB')
|
|
]
|
|
sstruct = struct.Struct('<' + ''.join([s.format[1:].decode() for s in _dir_item]))
|
|
|
|
def __init__(self, data, pos):
|
|
next_pos = pos + DiskKey.sstruct.size
|
|
self.location = DiskKey(data[pos:next_pos])
|
|
pos = next_pos
|
|
self.transid, self.data_len, self.name_len, self.type = \
|
|
DirItem._dir_item[1].unpack_from(data, pos)
|
|
pos += DirItem._dir_item[1].size
|
|
self.name, = struct.Struct('<{}s'.format(self.name_len)).unpack_from(data, pos)
|
|
pos += self.name_len
|
|
self.data, = struct.Struct('<{}s'.format(self.data_len)).unpack_from(data, pos)
|
|
pos += self.data_len
|
|
self._len = DirItem.sstruct.size + self.name_len + self.data_len
|
|
|
|
@property
|
|
def type_str(self):
|
|
return _dir_item_type_str_map[self.type]
|
|
|
|
@property
|
|
def name_str(self):
|
|
return embedded_text_for_str(self.name)
|
|
|
|
@property
|
|
def data_str(self):
|
|
return embedded_text_for_str(self.data)
|
|
|
|
def __len__(self):
|
|
return self._len
|
|
|
|
def __str__(self):
|
|
return "dir item location {self.location} type {self.type_str} " \
|
|
"name {self.name_str}".format(self=self)
|
|
|
|
|
|
class XAttrItem(DirItem):
|
|
def __str__(self):
|
|
return "xattr item name {self.name_str} data {self.data_str}".format(self=self)
|
|
|
|
|
|
class DirIndex(ItemData):
|
|
def __init__(self, header, data):
|
|
super().__init__(header)
|
|
self.location = DiskKey(data[:DiskKey.sstruct.size])
|
|
pos = DiskKey.sstruct.size
|
|
self.transid, self.data_len, self.name_len, self.type = \
|
|
DirItem._dir_item[1].unpack_from(data, pos)
|
|
pos += DirItem._dir_item[1].size
|
|
self.name, = struct.Struct('<{}s'.format(self.name_len)).unpack_from(data, pos)
|
|
|
|
@property
|
|
def type_str(self):
|
|
return _dir_item_type_str_map[self.type]
|
|
|
|
@property
|
|
def name_str(self):
|
|
return embedded_text_for_str(self.name)
|
|
|
|
def __str__(self):
|
|
return "dir index {self.key.offset} location {self.location} type {self.type_str} " \
|
|
"name {self.name_str}".format(self=self)
|
|
|
|
|
|
class FileExtentItem(ItemData):
|
|
_file_extent_item = [
|
|
struct.Struct('<QQBB2xB'),
|
|
struct.Struct('<4Q'),
|
|
]
|
|
sstruct = struct.Struct('<' + ''.join([s.format[1:].decode()
|
|
for s in _file_extent_item]))
|
|
|
|
def __init__(self, key, data):
|
|
super().__init__(key)
|
|
self.logical_offset = key.offset
|
|
self.generation, self.ram_bytes, self.compression, self.encryption, self.type = \
|
|
FileExtentItem._file_extent_item[0].unpack_from(data)
|
|
if self.type != FILE_EXTENT_INLINE:
|
|
# These are confusing, so they deserve a comment in the code:
|
|
# (disk_bytenr EXTENT_ITEM disk_num_bytes) is the tree key of
|
|
# the extent item storing the actual data.
|
|
#
|
|
# The third one, offset is the offset inside that extent where the
|
|
# data we need starts. num_bytes is the amount of bytes to be used
|
|
# from that offset onwards.
|
|
#
|
|
# Remember that these numbers always be multiples of disk block
|
|
# sizes, because that's how it gets cowed. We don't just use 1 or 2
|
|
# bytes from another extent.
|
|
pos = FileExtentItem._file_extent_item[0].size
|
|
self.disk_bytenr, self.disk_num_bytes, self.offset, self.num_bytes = \
|
|
FileExtentItem._file_extent_item[1].unpack_from(data, pos)
|
|
else:
|
|
self._inline_encoded_nbytes = key.data_size - FileExtentItem._file_extent_item[0].size
|
|
|
|
@property
|
|
def compression_str(self):
|
|
return _compress_type_str_map.get(self.compression, 'unknown')
|
|
|
|
@property
|
|
def type_str(self):
|
|
return _file_extent_type_str_map.get(self.type, 'unknown')
|
|
|
|
def __str__(self):
|
|
ret = ["extent data at {self.logical_offset} generation {self.generation} "
|
|
"ram_bytes {self.ram_bytes} "
|
|
"compression {self.compression_str} type {self.type_str}".format(self=self)]
|
|
if self.type != FILE_EXTENT_INLINE:
|
|
ret.append("disk_bytenr {self.disk_bytenr} disk_num_bytes {self.disk_num_bytes} "
|
|
"offset {self.offset} num_bytes {self.num_bytes}".format(self=self))
|
|
else:
|
|
ret.append("inline_encoded_nbytes {self._inline_encoded_nbytes}".format(self=self))
|
|
return ' '.join(ret)
|
|
|
|
|
|
class ExtentItem(ItemData):
|
|
sstruct = struct.Struct('<3Q')
|
|
extent_inline_ref = struct.Struct('<BQ')
|
|
|
|
def __init__(self, header, data, load_data_refs=True, load_metadata_refs=True):
|
|
super().__init__(header)
|
|
self.setattr_from_key(objectid_attr='vaddr', offset_attr='length')
|
|
pos = 0
|
|
self.refs, self.generation, self.flags = ExtentItem.sstruct.unpack_from(data, pos)
|
|
pos += ExtentItem.sstruct.size
|
|
if self.flags == EXTENT_FLAG_DATA and load_data_refs:
|
|
self.extent_data_refs = []
|
|
self.shared_data_refs = []
|
|
while pos < len(data):
|
|
inline_ref_type, inline_ref_offset = \
|
|
ExtentItem.extent_inline_ref.unpack_from(data, pos)
|
|
if inline_ref_type == EXTENT_DATA_REF_KEY:
|
|
pos += 1
|
|
next_pos = pos + InlineExtentDataRef.sstruct.size
|
|
self.extent_data_refs.append(InlineExtentDataRef(data[pos:next_pos]))
|
|
pos = next_pos
|
|
elif inline_ref_type == SHARED_DATA_REF_KEY:
|
|
pos += 1
|
|
next_pos = pos + InlineSharedDataRef.inline_shared_data_ref.size
|
|
self.shared_data_refs.append(InlineSharedDataRef(data[pos:next_pos]))
|
|
pos = next_pos
|
|
elif self.flags & EXTENT_FLAG_TREE_BLOCK and load_metadata_refs:
|
|
next_pos = pos + TreeBlockInfo.tree_block_info.size
|
|
self.tree_block_info = TreeBlockInfo(data[pos:next_pos])
|
|
pos = next_pos
|
|
self.tree_block_refs = []
|
|
self.shared_block_refs = []
|
|
while pos < len(data):
|
|
inline_ref_type, inline_ref_offset = \
|
|
ExtentItem.extent_inline_ref.unpack_from(data, pos)
|
|
if inline_ref_type == TREE_BLOCK_REF_KEY:
|
|
self.tree_block_refs.append(InlineTreeBlockRef(inline_ref_offset))
|
|
elif inline_ref_type == SHARED_BLOCK_REF_KEY:
|
|
self.shared_block_refs.append(InlineSharedBlockRef(inline_ref_offset))
|
|
else:
|
|
raise Exception("BUG: expected inline TREE_BLOCK_REF or SHARED_BLOCK_REF_KEY "
|
|
"but got inline_ref_type {}".format(inline_ref_type))
|
|
pos += ExtentItem.extent_inline_ref.size
|
|
|
|
def append_extent_data_ref(self, ref):
|
|
self.extent_data_refs.append(ref)
|
|
|
|
def append_shared_data_ref(self, ref):
|
|
self.shared_data_refs.append(ref)
|
|
|
|
def append_tree_block_ref(self, ref):
|
|
self.tree_block_refs.append(ref)
|
|
|
|
def append_shared_block_ref(self, ref):
|
|
self.shared_block_refs.append(ref)
|
|
|
|
@property
|
|
def flags_str(self):
|
|
return flags_str(self.flags, _extent_flags_str_map)
|
|
|
|
def __str__(self):
|
|
return "extent vaddr {self.vaddr} length {self.length} refs {self.refs} " \
|
|
"gen {self.generation} flags {self.flags_str}".format(self=self)
|
|
|
|
|
|
class ExtentDataRef(ItemData):
|
|
sstruct = struct.Struct('<3QL')
|
|
|
|
def __init__(self, header, data):
|
|
super().__init__(header)
|
|
self.root, self.objectid, self.offset, self.count = \
|
|
ExtentDataRef.sstruct.unpack(data)
|
|
|
|
def __str__(self):
|
|
return "extent data backref root {self.root} objectid {self.objectid} " \
|
|
"offset {self.offset} count {self.count}".format(self=self)
|
|
|
|
|
|
class InlineExtentDataRef(ExtentDataRef):
|
|
sstruct = ExtentDataRef.sstruct
|
|
|
|
def __init__(self, data):
|
|
self.root, self.objectid, self.offset, self.count = \
|
|
InlineExtentDataRef.sstruct.unpack(data)
|
|
|
|
def __str__(self):
|
|
return "inline extent data backref root {self.root} objectid {self.objectid} " \
|
|
"offset {self.offset} count {self.count}".format(self=self)
|
|
|
|
|
|
class SharedDataRef(ItemData):
|
|
sstruct = struct.Struct('<L')
|
|
|
|
def __init__(self, header, data):
|
|
super().__init__(header)
|
|
self.setattr_from_key(offset_attr='parent')
|
|
self.count, = SharedDataRef.sstruct.unpack(data)
|
|
|
|
def __str__(self):
|
|
return "shared data backref parent {self.parent} count {self.count}".format(self=self)
|
|
|
|
|
|
class InlineSharedDataRef(SharedDataRef):
|
|
sstruct = struct.Struct('<QL')
|
|
|
|
def __init__(self, data):
|
|
self.parent, self.count = InlineSharedDataRef.sstruct.unpack(data)
|
|
|
|
def __str__(self):
|
|
return "inline shared data backref parent {self.parent} " \
|
|
"count {self.count}".format(self=self)
|
|
|
|
|
|
class TreeBlockInfo(object):
|
|
sstruct = struct.Struct('<QBQB')
|
|
|
|
def __init__(self, data):
|
|
tb_objectid, tb_type, tb_offset, self.level = \
|
|
TreeBlockInfo.sstruct.unpack(data)
|
|
self.key = Key(tb_objectid, tb_type, tb_offset)
|
|
|
|
def __str__(self):
|
|
return "tree block key {self.key} level {self.level}".format(self=self)
|
|
|
|
class TreeBlockRef(ItemData):
|
|
def __init__(self, header):
|
|
super().__init__(header)
|
|
self.setattr_from_key(offset_attr='root')
|
|
|
|
def __str__(self):
|
|
return "tree block backref root {}".format(key_objectid_str(self.root, None))
|
|
|
|
|
|
class InlineTreeBlockRef(TreeBlockRef):
|
|
def __init__(self, root):
|
|
self.root = root
|
|
|
|
def __str__(self):
|
|
return "inline tree block backref root {}".format(key_objectid_str(self.root, None))
|
|
|
|
|
|
class SharedBlockRef(ItemData):
|
|
def __init__(self, header):
|
|
super().__init__(header)
|
|
self.setattr_from_key(offset_attr='parent')
|
|
|
|
def __str__(self):
|
|
return "shared block backref parent {}".format(self.parent)
|
|
|
|
|
|
class InlineSharedBlockRef(SharedBlockRef):
|
|
def __init__(self, parent):
|
|
self.parent = parent
|
|
|
|
def __str__(self):
|
|
return "inline shared block backref parent {}".format(self.parent)
|
|
|
|
# === Main FileSystem class
|
|
|
|
def key_bin_search(fd, base_offset, item_size, cmp_item, min, max):
|
|
low = min
|
|
high = max
|
|
|
|
while low < high:
|
|
mid = (low + high) // 2
|
|
offset = base_offset + mid * item_size
|
|
|
|
fd.seek(offset)
|
|
key1 = DiskKey(fd.read(item_size))
|
|
if key1 > cmp_item:
|
|
high = mid
|
|
elif key1 < cmp_item:
|
|
low = mid + 1
|
|
else:
|
|
return True, mid
|
|
|
|
return False, low
|
|
|
|
|
|
chunk_map_item = namedtuple('chunk_map_item', 'logical physical length devid')
|
|
|
|
|
|
class FileSystem(object):
|
|
def __init__(self, path, part_offset):
|
|
self._chunk_map = OrderedDict()
|
|
self.path = path
|
|
self.part_offset = part_offset
|
|
self.fd = open(path, 'rb')
|
|
self.fd.seek(part_offset + 0x10000) # going to superblock
|
|
|
|
sb_bytes = self.fd.read(superblock.size)
|
|
sb_tuple = superblock.unpack(sb_bytes)
|
|
|
|
if sb_tuple[3] != b'_BHRfS_M':
|
|
raise "No signature found"
|
|
|
|
# setting base FS information
|
|
self.fsid = sb_tuple[0]
|
|
self.nodesize = sb_tuple[14]
|
|
self.sectorsize = sb_tuple[13]
|
|
|
|
self._chunk_root = sb_tuple[6]
|
|
self._chunk_root_level = sb_tuple[24]
|
|
self._tree_roots_root = sb_tuple[5]
|
|
self._tree_roots_root_level = sb_tuple[23]
|
|
|
|
# setting chunk map
|
|
sys_chunk_array_size = sb_tuple[17]
|
|
sys_chunk = sb_tuple[25][:sys_chunk_array_size]
|
|
|
|
pos = 0
|
|
while pos < sys_chunk_array_size:
|
|
key = DiskKey(sys_chunk[pos:])
|
|
pos += DiskKey.sstruct.size
|
|
chunk = Chunk(key, sys_chunk[pos:])
|
|
for st in chunk.stripes:
|
|
self._insert_chunk(chunk_map_item(chunk.vaddr, st.offset, chunk.length, st.devid))
|
|
pos += chunk.size
|
|
|
|
# setting tree roots
|
|
_, fs_tree_root_item = self.search_tree(self._tree_roots_root_level, self._tree_roots_root, Key(FS_TREE_OBJECTID, ROOT_ITEM_KEY, 0))
|
|
_, extent_tree_root_item = self.search_tree(self._tree_roots_root_level, self._tree_roots_root, Key(EXTENT_TREE_OBJECTID, ROOT_ITEM_KEY, 0))
|
|
self._fs_root_level = fs_tree_root_item.level
|
|
self._fs_root = fs_tree_root_item.bytenr
|
|
self._extent_root_level = extent_tree_root_item.level
|
|
self._extent_root = extent_tree_root_item.bytenr
|
|
|
|
@property
|
|
def chunk_root(self):
|
|
return self._chunk_root_level, self._chunk_root
|
|
|
|
@property
|
|
def tree_roots_root(self):
|
|
return self._tree_roots_root_level, self._tree_roots_root
|
|
|
|
@property
|
|
def fs_root(self):
|
|
return self._fs_root_level, self._fs_root
|
|
|
|
@property
|
|
def extent_root(self):
|
|
return self._extent_root_level, self._extent_root
|
|
|
|
def logical_to_physical(self, log):
|
|
cur_logical = next(iter(self._chunk_map)) # first item
|
|
|
|
for logical, cmi in self._chunk_map.items():
|
|
if logical > log:
|
|
break
|
|
cur_logical = logical
|
|
|
|
# if there is no address in chunk_map, searching in chunk_tree
|
|
if cur_logical + self._chunk_map[cur_logical].length < log:
|
|
def process_func(header, offset):
|
|
for i in range(header.items_num):
|
|
self.fd.seek(offset + i * LeafKey.sstruct.size)
|
|
k = LeafKey(self.fd.read(LeafKey.sstruct.size))
|
|
self.fd.seek(offset + k.data_offset)
|
|
if k.type == CHUNK_ITEM_KEY:
|
|
item = _key_type_class_map[k.type](k, self.fd.read(k.data_size))
|
|
for st in item.stripes:
|
|
self._insert_chunk(chunk_map_item(item.vaddr, st.offset, item.length, st.devid))
|
|
|
|
self.search_tree(self._chunk_root_level, self._chunk_root, Key(FIRST_CHUNK_TREE_OBJECTID, CHUNK_ITEM_KEY, log), process_func)
|
|
|
|
cur_logical = next(iter(self._chunk_map)) # first item
|
|
if cur_logical > log:
|
|
raise Exception(f'Cannot translate address {log:#x}')
|
|
|
|
for logical, cmi in self._chunk_map.items():
|
|
if logical > log:
|
|
break
|
|
cur_logical = logical
|
|
|
|
if cur_logical + self._chunk_map[cur_logical].length < log:
|
|
raise Exception(f'Cannot translate address {log:#x}')
|
|
|
|
print('address translation: {:#x} -> {:#x}'.format(log, self._chunk_map[cur_logical].physical + log - cur_logical))
|
|
return self.part_offset + self._chunk_map[cur_logical].physical + log - cur_logical
|
|
|
|
def search_tree(self, level, root_offset, key, process_node_func = None):
|
|
for lvl in range(level, 0, -1):
|
|
# inner node
|
|
root_offset = self.logical_to_physical(root_offset)
|
|
self.fd.seek(root_offset)
|
|
header = NodeHeader._make(_node_header_struct.unpack(self.fd.read(_node_header_struct.size)))
|
|
if header.level != lvl:
|
|
raise Exception('Invalid inner node level')
|
|
|
|
found, itemnr = key_bin_search(
|
|
self.fd,
|
|
root_offset + _node_header_struct.size,
|
|
InnerKey.sstruct.size,
|
|
key,
|
|
0,
|
|
header.items_num
|
|
)
|
|
|
|
# TODO: better understand this
|
|
if not found and itemnr > 0:
|
|
itemnr -= 1
|
|
|
|
self.fd.seek(root_offset + _node_header_struct.size + itemnr * InnerKey.sstruct.size)
|
|
k = InnerKey(self.fd.read(InnerKey.sstruct.size))
|
|
root_offset = k.block_num
|
|
else:
|
|
# we are in leaf node
|
|
root_offset = self.logical_to_physical(root_offset)
|
|
self.fd.seek(root_offset)
|
|
header = NodeHeader._make(_node_header_struct.unpack(self.fd.read(_node_header_struct.size)))
|
|
if header.level != 0:
|
|
raise Exception('Invalid leaf level')
|
|
|
|
if process_node_func:
|
|
process_node_func(header, root_offset + _node_header_struct.size)
|
|
return root_offset
|
|
|
|
found, itemnr = key_bin_search(
|
|
self.fd,
|
|
root_offset + _node_header_struct.size,
|
|
LeafKey.sstruct.size,
|
|
key,
|
|
0,
|
|
header.items_num
|
|
)
|
|
|
|
self.fd.seek(root_offset + _node_header_struct.size + itemnr * LeafKey.sstruct.size)
|
|
k = LeafKey(self.fd.read(LeafKey.sstruct.size))
|
|
self.fd.seek(root_offset + _node_header_struct.size + k.data_offset)
|
|
|
|
if k.type in _key_type_class_map:
|
|
return k, _key_type_class_map[k.type](k, self.fd.read(k.data_size))
|
|
else:
|
|
return k, False
|
|
|
|
def print_node(self, header, offset):
|
|
print(header)
|
|
root_paddr = self.logical_to_physical(header.node_addr)
|
|
key_size = InnerKey.sstruct.size if header.level > 0 else LeafKey.sstruct.size
|
|
key_struct = InnerKey if header.level > 0 else LeafKey
|
|
for i in range(header.items_num):
|
|
self.fd.seek(root_paddr + _node_header_struct.size + i * key_size)
|
|
k = key_struct(self.fd.read(key_size))
|
|
print(k)
|
|
self.fd.seek(root_paddr + _node_header_struct.size + k.data_offset)
|
|
if k.type in _key_type_class_map and header.level == 0:
|
|
item = _key_type_class_map[k.type](k, self.fd.read(k.data_size))
|
|
print(item)
|
|
if k.type == DIR_ITEM_KEY:
|
|
for it in item:
|
|
print(it)
|
|
print('============================')
|
|
|
|
def print_chunk_map(self):
|
|
print('=== chunk map ===')
|
|
for logical, cmi in self._chunk_map.items():
|
|
print(f'{cmi.logical:#x}..{cmi.logical+cmi.length:#x} -> {cmi.physical:#x}..{cmi.physical+cmi.length:#x}')
|
|
print('=================')
|
|
|
|
def _insert_chunk(self, chunk):
|
|
if not chunk.logical in self._chunk_map:
|
|
cm = dict(self._chunk_map)
|
|
cm[chunk.logical] = chunk
|
|
self._chunk_map = OrderedDict(sorted(cm.items()))
|
|
|
|
|
|
_key_type_class_map = {
|
|
INODE_ITEM_KEY: InodeItem,
|
|
INODE_REF_KEY: InodeRef,
|
|
DIR_ITEM_KEY: DirItemList,
|
|
DIR_INDEX_KEY: DirIndex,
|
|
EXTENT_DATA_KEY: FileExtentItem,
|
|
ROOT_ITEM_KEY: RootItem,
|
|
EXTENT_ITEM_KEY: ExtentItem,
|
|
CHUNK_ITEM_KEY: Chunk,
|
|
}
|