mirror of
https://github.com/reactos/reactos.git
synced 2024-10-30 19:41:57 +00:00
505 lines
17 KiB
Python
505 lines
17 KiB
Python
'''
|
|
PROJECT: ReactOS apisets generator
|
|
LICENSE: MIT (https://spdx.org/licenses/MIT)
|
|
PURPOSE: Create apiset forwarders based on Wine apisets
|
|
COPYRIGHT: Copyright 2017,2018 Mark Jansen (mark.jansen@reactos.org)
|
|
'''
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
from collections import defaultdict
|
|
import subprocess
|
|
|
|
|
|
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
NL_CHAR = '\n'
|
|
|
|
IGNORE_OPTIONS = ('-norelay', '-ret16', '-ret64', '-register', '-private',
|
|
'-noname', '-ordinal', '-i386', '-arch=', '-stub', '-version=')
|
|
|
|
# Figure these out later
|
|
FUNCTION_BLACKLIST = [
|
|
# api-ms-win-crt-utility-l1-1-0_stubs.c(6):
|
|
# error C2169: '_abs64': intrinsic function, cannot be defined
|
|
'_abs64',
|
|
'_byteswap_uint64', '_byteswap_ulong', '_byteswap_ushort',
|
|
'_rotl64', '_rotr64',
|
|
]
|
|
|
|
SPEC_HEADER = [
|
|
'\n',
|
|
'# This file is autogenerated by update.py\n',
|
|
'\n'
|
|
]
|
|
|
|
ALIAS_DLL = {
|
|
'ucrtbase': 'msvcrt',
|
|
'kernelbase': 'kernel32',
|
|
'shcore': 'shlwapi',
|
|
'combase': 'ole32',
|
|
|
|
# These modules cannot be linked against in ROS, so forward it
|
|
'cfgmgr32': 'setupapi', # Forward everything
|
|
'wmi': 'advapi32', # Forward everything
|
|
}
|
|
|
|
class InvalidSpecError(Exception):
|
|
def __init__(self, message):
|
|
Exception.__init__(self, message)
|
|
|
|
class Arch(object):
|
|
none = 0
|
|
i386 = 1
|
|
x86_64 = 2
|
|
arm = 4
|
|
arm64 = 8
|
|
Any = i386 | x86_64 | arm | arm64
|
|
|
|
FROM_STR = {
|
|
'i386': i386,
|
|
'x86_64': x86_64,
|
|
'arm': arm,
|
|
'arm64': arm64,
|
|
'any': Any,
|
|
'win32': i386,
|
|
'win64': x86_64,
|
|
}
|
|
|
|
TO_STR = {
|
|
i386: 'i386',
|
|
x86_64: 'x86_64',
|
|
arm: 'arm',
|
|
arm64: 'arm64',
|
|
}
|
|
|
|
def __init__(self, initial=none):
|
|
self._val = initial
|
|
|
|
def add(self, text):
|
|
self._val |= sum([Arch.FROM_STR[arch] for arch in text.split(',')])
|
|
assert self._val != 0
|
|
|
|
def has(self, val):
|
|
return (self._val & val) != 0
|
|
|
|
def to_str(self):
|
|
arch_str = []
|
|
for value in Arch.TO_STR:
|
|
if value & self._val:
|
|
arch_str.append(Arch.TO_STR[value])
|
|
return ','.join(arch_str)
|
|
|
|
def __len__(self):
|
|
return bin(self._val).count("1")
|
|
|
|
def __add__(self, other):
|
|
return Arch(self._val | other._val) # pylint: disable=W0212
|
|
|
|
def __sub__(self, other):
|
|
return Arch(self._val & ~other._val) # pylint: disable=W0212
|
|
|
|
def __gt__(self, other):
|
|
return self._val > other._val # pylint: disable=W0212
|
|
|
|
def __lt__(self, other):
|
|
return self._val < other._val # pylint: disable=W0212
|
|
|
|
def __eq__(self, other):
|
|
return self._val == other._val # pylint: disable=W0212
|
|
|
|
def __ne__(self, other):
|
|
return not self.__eq__(other)
|
|
|
|
|
|
class VersionExpr(object):
|
|
def __init__(self, text):
|
|
self.text = text
|
|
self.parse()
|
|
|
|
def parse(self):
|
|
pass
|
|
|
|
def to_str(self):
|
|
if self.text:
|
|
return '-version={}'.format(self.text)
|
|
return ''
|
|
|
|
|
|
class SpecEntry(object):
|
|
def __init__(self, text, spec):
|
|
self.spec = spec
|
|
self._ord = None
|
|
self.callconv = None
|
|
self.name = None
|
|
self.arch = Arch()
|
|
self.version = None
|
|
self._forwarder = None
|
|
self.init(text)
|
|
self.noname = False
|
|
if self.name == '@':
|
|
self.noname = True
|
|
if self._forwarder:
|
|
self.name = self._forwarder[1]
|
|
|
|
def init(self, text):
|
|
tokens = re.split(r'([\s\(\)#;])', text.strip())
|
|
tokens = [token for token in tokens if token and not token.isspace()]
|
|
idx = []
|
|
for comment in ['#', ';']:
|
|
if comment in tokens:
|
|
idx.append(tokens.index(comment))
|
|
idx = sorted(idx)
|
|
if idx:
|
|
tokens = tokens[:idx[0]]
|
|
if not tokens:
|
|
raise InvalidSpecError(text)
|
|
self._ord = tokens[0]
|
|
assert self._ord == '@' or self._ord.isdigit(), text
|
|
tokens = tokens[1:]
|
|
self.callconv = tokens.pop(0)
|
|
self.name = tokens.pop(0)
|
|
while self.name.startswith(IGNORE_OPTIONS):
|
|
if self.name.startswith('-arch='):
|
|
self.arch.add(self.name[6:])
|
|
elif self.name.startswith('-version='):
|
|
self.version = VersionExpr(self.name[9:])
|
|
elif self.name == '-i386':
|
|
self.arch.add('i386')
|
|
self.name = tokens.pop(0)
|
|
if not self.arch:
|
|
self.arch = Arch(Arch.Any)
|
|
assert not self.name.startswith('-'), text
|
|
if not tokens:
|
|
return
|
|
if tokens[0] == '(':
|
|
assert ')' in tokens, text
|
|
arg = tokens.pop(0)
|
|
while True:
|
|
arg = tokens.pop(0)
|
|
if arg == ')':
|
|
break
|
|
if not tokens:
|
|
return
|
|
assert len(tokens) == 1, text
|
|
self._forwarder = tokens.pop(0).split('.', 2)
|
|
if len(self._forwarder) == 1:
|
|
self._forwarder = ['self', self._forwarder[0]]
|
|
assert len(self._forwarder) in (0, 2), self._forwarder
|
|
if self._forwarder[0] in ALIAS_DLL:
|
|
self._forwarder[0] = ALIAS_DLL[self._forwarder[0]]
|
|
|
|
def resolve_forwarders(self, module_lookup, try_modules):
|
|
if self._forwarder:
|
|
assert self._forwarder[1] == self.name, '{}:{}'.format(self._forwarder[1], self.name)
|
|
if self.noname and self.name == '@':
|
|
return 0 # cannot search for this function
|
|
self._forwarder = []
|
|
self.arch = Arch()
|
|
for module_name in try_modules:
|
|
assert module_name in module_lookup, module_name
|
|
module = module_lookup[module_name]
|
|
fwd_arch = module.find_arch(self.name)
|
|
callconv = module.find_callconv(self.name)
|
|
version = module.find_version(self.name)
|
|
if fwd_arch:
|
|
self.arch = fwd_arch
|
|
self._forwarder = [module_name, self.name]
|
|
self.callconv = callconv
|
|
self.version = version
|
|
return 1
|
|
return 0
|
|
|
|
def extra_forwarders(self, function_lookup, module_lookup):
|
|
if self._forwarder:
|
|
return 1
|
|
if self.noname and self.name == '@':
|
|
return 0 # cannot search for this function
|
|
lst = function_lookup.get(self.name, None)
|
|
if lst:
|
|
modules = list(set([func.spec.name for func in lst]))
|
|
if len(modules) > 1:
|
|
mod = None
|
|
arch = Arch()
|
|
for module in modules:
|
|
mod_arch = module_lookup[module].find_arch(self.name)
|
|
if mod is None or mod_arch > arch:
|
|
mod = module
|
|
arch = mod_arch
|
|
modules = [mod]
|
|
mod = modules[0]
|
|
self._forwarder = [mod, self.name]
|
|
mod = module_lookup[mod]
|
|
self.arch = mod.find_arch(self.name)
|
|
self.callconv = mod.find_callconv(self.name)
|
|
self.version = mod.find_version(self.name)
|
|
return 1
|
|
return 0
|
|
|
|
def forwarder_module(self):
|
|
if self._forwarder:
|
|
return self._forwarder[0]
|
|
|
|
def forwarder(self):
|
|
if self._forwarder:
|
|
return 1
|
|
return 0
|
|
|
|
def write(self, spec_file):
|
|
name = self.name
|
|
opts = ''
|
|
estimate_size = 0
|
|
if self.noname:
|
|
opts = '{} -noname'.format(opts)
|
|
if self.version:
|
|
opts = '{} {}'.format(opts, self.version.to_str())
|
|
if self.name == '@':
|
|
assert self._ord != '@'
|
|
name = 'Ordinal' + self._ord
|
|
if not self._forwarder:
|
|
spec_file.write('{} stub{} {}{}'.format(self._ord, opts, name, NL_CHAR))
|
|
estimate_size += 0x1000
|
|
else:
|
|
assert self.arch != Arch(), self.name
|
|
args = '()'
|
|
callconv = 'stdcall'
|
|
fwd = '.'.join(self._forwarder)
|
|
name = self.name if not self.noname else '@'
|
|
arch = self.arch
|
|
if self.callconv == 'extern':
|
|
args = ''
|
|
callconv = 'extern -stub' # HACK
|
|
fwd += ' # the -stub is a HACK to fix VS < 2017 build!'
|
|
if arch != Arch(Arch.Any):
|
|
opts = '{} -arch={}'.format(opts, arch.to_str())
|
|
spec_file.write('{ord} {cc}{opts} {name}{args} {fwd}{nl}'.format(ord=self._ord,
|
|
cc=callconv,
|
|
opts=opts,
|
|
name=name,
|
|
args=args,
|
|
fwd=fwd,
|
|
nl=NL_CHAR))
|
|
estimate_size += 0x100
|
|
return estimate_size
|
|
|
|
|
|
|
|
class SpecFile(object):
|
|
def __init__(self, fullpath, name):
|
|
self._path = fullpath
|
|
self.name = name
|
|
self._entries = []
|
|
self._functions = defaultdict(list)
|
|
self._estimate_size = 0
|
|
|
|
def parse(self):
|
|
with open(self._path, 'rb') as specfile:
|
|
for line in specfile.readlines():
|
|
if line:
|
|
try:
|
|
entry = SpecEntry(line, self)
|
|
self._entries.append(entry)
|
|
self._functions[entry.name].append(entry)
|
|
except InvalidSpecError:
|
|
pass
|
|
return (sum([entry.forwarder() for entry in self._entries]), len(self._entries))
|
|
|
|
def add_functions(self, function_lookup):
|
|
for entry in self._entries:
|
|
function_lookup[entry.name].append(entry)
|
|
|
|
def find(self, name):
|
|
return self._functions.get(name, None)
|
|
|
|
def find_arch(self, name):
|
|
functions = self.find(name)
|
|
arch = Arch()
|
|
if functions:
|
|
for func in functions:
|
|
arch += func.arch
|
|
return arch
|
|
|
|
def find_callconv(self, name):
|
|
functions = self.find(name)
|
|
callconv = None
|
|
if functions:
|
|
for func in functions:
|
|
if not callconv:
|
|
callconv = func.callconv
|
|
elif callconv != func.callconv:
|
|
assert callconv != 'extern', 'Cannot have data/function with same name'
|
|
callconv = func.callconv
|
|
return callconv
|
|
|
|
def find_version(self, name):
|
|
functions = self.find(name)
|
|
version = None
|
|
if functions:
|
|
for func in functions:
|
|
if not func.version:
|
|
continue
|
|
if version:
|
|
assert version.text == func.version.text
|
|
version = func.version
|
|
return version
|
|
|
|
def resolve_forwarders(self, module_lookup):
|
|
modules = self.forwarder_modules()
|
|
total = 0
|
|
for entry in self._entries:
|
|
total += entry.resolve_forwarders(module_lookup, modules)
|
|
return (total, len(self._entries))
|
|
|
|
def extra_forwarders(self, function_lookup, module_lookup):
|
|
total = 0
|
|
for entry in self._entries:
|
|
total += entry.extra_forwarders(function_lookup, module_lookup)
|
|
return (total, len(self._entries))
|
|
|
|
def forwarder_modules(self):
|
|
modules = defaultdict(int)
|
|
for entry in self._entries:
|
|
module = entry.forwarder_module()
|
|
if module:
|
|
modules[module] += 1
|
|
return sorted(modules, key=modules.get, reverse=True)
|
|
|
|
def write(self, spec_file):
|
|
written = set(FUNCTION_BLACKLIST)
|
|
self._estimate_size = 0
|
|
for entry in self._entries:
|
|
if entry.name not in written:
|
|
self._estimate_size += entry.write(spec_file)
|
|
written.add(entry.name)
|
|
|
|
def write_cmake(self, cmakelists, baseaddress):
|
|
seen = set()
|
|
# ntdll and kernel32 are linked against everything, self = internal,
|
|
# we cannot link cfgmgr32 and wmi?
|
|
ignore = ['ntdll', 'kernel32', 'self', 'cfgmgr32', 'wmi']
|
|
forwarders = self.forwarder_modules()
|
|
fwd_strings = [x for x in forwarders if not (x in seen or x in ignore or seen.add(x))]
|
|
fwd_strings = ' '.join(fwd_strings)
|
|
name = self.name
|
|
baseaddress = '0x{:8x}'.format(baseaddress)
|
|
cmakelists.write('add_apiset({} {} {}){}'.format(name, baseaddress, fwd_strings, NL_CHAR))
|
|
return self._estimate_size
|
|
|
|
|
|
|
|
def generate_specnames(dll_dir):
|
|
win32 = os.path.join(dll_dir, 'win32')
|
|
for dirname in os.listdir(win32):
|
|
fullpath = os.path.join(win32, dirname, dirname + '.spec')
|
|
if not os.path.isfile(fullpath):
|
|
if '.' in dirname:
|
|
fullpath = os.path.join(win32, dirname, dirname.rsplit('.', 1)[0] + '.spec')
|
|
if not os.path.isfile(fullpath):
|
|
continue
|
|
else:
|
|
continue
|
|
yield (fullpath, dirname)
|
|
# Special cases
|
|
yield (os.path.join(dll_dir, 'ntdll', 'def', 'ntdll.spec'), 'ntdll')
|
|
yield (os.path.join(dll_dir, 'appcompat', 'apphelp', 'apphelp.spec'), 'apphelp')
|
|
yield (os.path.join(dll_dir, '..', 'win32ss', 'user', 'user32', 'user32.spec'), 'user32')
|
|
yield (os.path.join(dll_dir, '..', 'win32ss', 'gdi', 'gdi32', 'gdi32.spec'), 'gdi32')
|
|
|
|
def run(wineroot):
|
|
global NL_CHAR
|
|
wine_apisets = []
|
|
ros_modules = []
|
|
|
|
module_lookup = {}
|
|
function_lookup = defaultdict(list)
|
|
|
|
version = subprocess.check_output(["git", "describe"], cwd=wineroot).strip()
|
|
|
|
print 'Reading Wine apisets for', version
|
|
wine_apiset_path = os.path.join(wineroot, 'dlls')
|
|
for dirname in os.listdir(wine_apiset_path):
|
|
if not dirname.startswith('api-'):
|
|
continue
|
|
if not os.path.isdir(os.path.join(wine_apiset_path, dirname)):
|
|
continue
|
|
fullpath = os.path.join(wine_apiset_path, dirname, dirname + '.spec')
|
|
spec = SpecFile(fullpath, dirname)
|
|
wine_apisets.append(spec)
|
|
|
|
print 'Parsing Wine apisets,',
|
|
total = (0, 0)
|
|
for apiset in wine_apisets:
|
|
total = tuple(map(sum, zip(apiset.parse(), total)))
|
|
print 'found', total[0], '/', total[1], 'forwarders'
|
|
|
|
print 'Reading ReactOS modules'
|
|
for fullpath, dllname in generate_specnames(os.path.dirname(SCRIPT_DIR)):
|
|
spec = SpecFile(fullpath, dllname)
|
|
ros_modules.append(spec)
|
|
|
|
print 'Parsing ReactOS modules'
|
|
for module in ros_modules:
|
|
module.parse()
|
|
assert module.name not in module_lookup, module.name
|
|
module_lookup[module.name] = module
|
|
module.add_functions(function_lookup)
|
|
|
|
print 'First pass, resolving forwarders,',
|
|
total = (0, 0)
|
|
for apiset in wine_apisets:
|
|
total = tuple(map(sum, zip(apiset.resolve_forwarders(module_lookup), total)))
|
|
print 'found', total[0], '/', total[1], 'forwarders'
|
|
|
|
print 'Second pass, searching extra forwarders,',
|
|
total = (0, 0)
|
|
for apiset in wine_apisets:
|
|
total = tuple(map(sum, zip(apiset.extra_forwarders(function_lookup, module_lookup), total)))
|
|
print 'found', total[0], '/', total[1], 'forwarders'
|
|
|
|
with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt.in'), 'rb') as template:
|
|
cmake_template = template.read()
|
|
cmake_template = cmake_template.replace('%WINE_GIT_VERSION%', version)
|
|
# Detect the checkout newline settings
|
|
if '\r\n' in cmake_template:
|
|
NL_CHAR = '\r\n'
|
|
|
|
manifest_files = []
|
|
print 'Writing apisets'
|
|
spec_header = [line.replace('\n', NL_CHAR) for line in SPEC_HEADER]
|
|
for apiset in wine_apisets:
|
|
with open(os.path.join(SCRIPT_DIR, apiset.name + '.spec'), 'wb') as out_spec:
|
|
out_spec.writelines(spec_header)
|
|
apiset.write(out_spec)
|
|
manifest_files.append(' <file name="{}.dll"/>'.format(apiset.name))
|
|
|
|
print 'Generating manifest'
|
|
manifest_name = 'x86_reactos.apisets_6595b64144ccf1df_1.0.0.0_none_deadbeef.manifest'
|
|
with open(os.path.join(SCRIPT_DIR, manifest_name + '.in'), 'rb') as template:
|
|
manifest_template = template.read()
|
|
manifest_template = manifest_template.replace('%WINE_GIT_VERSION%', version)
|
|
file_list = '\r\n'.join(manifest_files)
|
|
manifest_template = manifest_template.replace('%MANIFEST_FILE_LIST%', file_list)
|
|
with open(os.path.join(SCRIPT_DIR, manifest_name), 'wb') as manifest:
|
|
manifest.write(manifest_template)
|
|
|
|
print 'Writing CMakeLists.txt'
|
|
baseaddress = 0x60000000
|
|
with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt'), 'wb') as cmakelists:
|
|
cmakelists.write(cmake_template)
|
|
for apiset in wine_apisets:
|
|
baseaddress += apiset.write_cmake(cmakelists, baseaddress)
|
|
baseaddress += (0x10000 - baseaddress) % 0x10000
|
|
print 'Done'
|
|
|
|
def main(paths):
|
|
for path in paths:
|
|
if path:
|
|
run(path)
|
|
return
|
|
print 'No path specified,'
|
|
print 'either pass it as argument, or set the environment variable "WINE_SRC_ROOT"'
|
|
|
|
if __name__ == '__main__':
|
|
main(sys.argv[1:] + [os.environ.get('WINE_SRC_ROOT')])
|