reactos/dll/apisets/update.py

'''
PROJECT:     ReactOS apisets generator
LICENSE:     MIT (https://spdx.org/licenses/MIT)
PURPOSE:     Create apiset forwarders based on Wine apisets
COPYRIGHT:   Copyright 2017,2018 Mark Jansen (mark.jansen@reactos.org)
'''

import os
import re
import sys
from collections import defaultdict
import subprocess


SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))

NL_CHAR = '\n'

IGNORE_OPTIONS = ('-norelay', '-ret16', '-ret64', '-register', '-private',
                  '-noname', '-ordinal', '-i386', '-arch=', '-stub', '-version=')

# Figure these out later
FUNCTION_BLACKLIST = [
    # api-ms-win-crt-utility-l1-1-0_stubs.c(6):
    # error C2169: '_abs64': intrinsic function, cannot be defined
    '_abs64',
    '_byteswap_uint64', '_byteswap_ulong', '_byteswap_ushort',
    '_rotl64', '_rotr64',
]

SPEC_HEADER = [
    '\n',
    '# This file is autogenerated by update.py\n',
    '\n'
]

ALIAS_DLL = {
    'ucrtbase': 'msvcrt',
    'kernelbase': 'kernel32',
    'shcore': 'shlwapi',
    'combase': 'ole32',

    # These modules cannot be linked against in ROS, so forward it
    'cfgmgr32': 'setupapi', # Forward everything
    'wmi': 'advapi32',      # Forward everything
}

class InvalidSpecError(Exception):
    def __init__(self, message):
        Exception.__init__(self, message)

class Arch(object):
    none = 0
    i386 = 1
    x86_64 = 2
    arm = 4
    arm64 = 8
    Any = i386 | x86_64 | arm | arm64

    FROM_STR = {
        'i386': i386,
        'x86_64': x86_64,
        'arm': arm,
        'arm64': arm64,
        'any': Any,
        'win32': i386,
        'win64': x86_64,
    }

    TO_STR = {
        i386: 'i386',
        x86_64: 'x86_64',
        arm: 'arm',
        arm64: 'arm64',
    }

    def __init__(self, initial=none):
        self._val = initial

    def add(self, text):
        self._val |= sum([Arch.FROM_STR[arch] for arch in text.split(',')])
        assert self._val != 0

    def has(self, val):
        return (self._val & val) != 0

    def to_str(self):
        arch_str = []
        for value in Arch.TO_STR:
            if value & self._val:
                arch_str.append(Arch.TO_STR[value])
        return ','.join(arch_str)

    def __len__(self):
        return bin(self._val).count("1")

    def __add__(self, other):
        return Arch(self._val | other._val) # pylint: disable=W0212

    def __sub__(self, other):
        return Arch(self._val & ~other._val) # pylint: disable=W0212

    def __gt__(self, other):
        return self._val > other._val       # pylint: disable=W0212

    def __lt__(self, other):
        return self._val < other._val       # pylint: disable=W0212

    def __eq__(self, other):
        return self._val == other._val      # pylint: disable=W0212

    def __ne__(self, other):
        return not self.__eq__(other)


class VersionExpr(object):
    def __init__(self, text):
        self.text = text
        self.parse()

    def parse(self):
        pass

    def to_str(self):
        if self.text:
            return '-version={}'.format(self.text)
        return ''


class SpecEntry(object):
    def __init__(self, text, spec):
        self.spec = spec
        self._ord = None
        self.callconv = None
        self.name = None
        self.arch = Arch()
        self.version = None
        self._forwarder = None
        self.init(text)
        self.noname = False
        if self.name == '@':
            self.noname = True
            if self._forwarder:
                self.name = self._forwarder[1]

    def init(self, text):
        tokens = re.split(r'([\s\(\)#;])', text.strip())
        tokens = [token for token in tokens if token and not token.isspace()]
        idx = []
        for comment in ['#', ';']:
            if comment in tokens:
                idx.append(tokens.index(comment))
        idx = sorted(idx)
        if idx:
            tokens = tokens[:idx[0]]
        if not tokens:
            raise InvalidSpecError(text)
        self._ord = tokens[0]
        assert self._ord == '@' or self._ord.isdigit(), text
        tokens = tokens[1:]
        self.callconv = tokens.pop(0)
        self.name = tokens.pop(0)
        while self.name.startswith(IGNORE_OPTIONS):
            if self.name.startswith('-arch='):
                self.arch.add(self.name[6:])
            elif self.name.startswith('-version='):
                self.version = VersionExpr(self.name[9:])
            elif self.name == '-i386':
                self.arch.add('i386')
            self.name = tokens.pop(0)
        if not self.arch:
            self.arch = Arch(Arch.Any)
        assert not self.name.startswith('-'), text
        if not tokens:
            return
        if tokens[0] == '(':
            assert ')' in tokens, text
            arg = tokens.pop(0)
            while True:
                arg = tokens.pop(0)
                if arg == ')':
                    break
        if not tokens:
            return
        assert len(tokens) == 1, text
        self._forwarder = tokens.pop(0).split('.', 2)
        if len(self._forwarder) == 1:
            self._forwarder = ['self', self._forwarder[0]]
        assert len(self._forwarder) in (0, 2), self._forwarder
        if self._forwarder[0] in ALIAS_DLL:
            self._forwarder[0] = ALIAS_DLL[self._forwarder[0]]

    def resolve_forwarders(self, module_lookup, try_modules):
        if self._forwarder:
            assert self._forwarder[1] == self.name, '{}:{}'.format(self._forwarder[1], self.name)
        if self.noname and self.name == '@':
            return 0    # cannot search for this function
        self._forwarder = []
        self.arch = Arch()
        for module_name in try_modules:
            assert module_name in module_lookup, module_name
            module = module_lookup[module_name]
            fwd_arch = module.find_arch(self.name)
            callconv = module.find_callconv(self.name)
            version = module.find_version(self.name)
            if fwd_arch:
                self.arch = fwd_arch
                self._forwarder = [module_name, self.name]
                self.callconv = callconv
                self.version = version
                return 1
        return 0

    def extra_forwarders(self, function_lookup, module_lookup):
        if self._forwarder:
            return 1
        if self.noname and self.name == '@':
            return 0    # cannot search for this function
        lst = function_lookup.get(self.name, None)
        if lst:
            modules = list(set([func.spec.name for func in lst]))
            if len(modules) > 1:
                mod = None
                arch = Arch()
                for module in modules:
                    mod_arch = module_lookup[module].find_arch(self.name)
                    if mod is None or mod_arch > arch:
                        mod = module
                        arch = mod_arch
                modules = [mod]
            mod = modules[0]
            self._forwarder = [mod, self.name]
            mod = module_lookup[mod]
            self.arch = mod.find_arch(self.name)
            self.callconv = mod.find_callconv(self.name)
            self.version = mod.find_version(self.name)
            return 1
        return 0

    def forwarder_module(self):
        if self._forwarder:
            return self._forwarder[0]

    def forwarder(self):
        if self._forwarder:
            return 1
        return 0

    def write(self, spec_file):
        name = self.name
        opts = ''
        estimate_size = 0
        if self.noname:
            opts = '{} -noname'.format(opts)
        if self.version:
            opts = '{} {}'.format(opts, self.version.to_str())
        if self.name == '@':
            assert self._ord != '@'
            name = 'Ordinal' + self._ord
        if not self._forwarder:
            spec_file.write('{} stub{} {}{}'.format(self._ord, opts, name, NL_CHAR))
            estimate_size += 0x1000
        else:
            assert self.arch != Arch(), self.name
            args = '()'
            callconv = 'stdcall'
            fwd = '.'.join(self._forwarder)
            name = self.name if not self.noname else '@'
            arch = self.arch
            if self.callconv == 'extern':
                args = ''
                callconv = 'extern -stub'   # HACK
                fwd += ' # the -stub is a HACK to fix VS < 2017 build!'
            if arch != Arch(Arch.Any):
                opts = '{} -arch={}'.format(opts, arch.to_str())
            spec_file.write('{ord} {cc}{opts} {name}{args} {fwd}{nl}'.format(ord=self._ord,
                                                                             cc=callconv,
                                                                             opts=opts,
                                                                             name=name,
                                                                             args=args,
                                                                             fwd=fwd,
                                                                             nl=NL_CHAR))
            estimate_size += 0x100
        return estimate_size



class SpecFile(object):
    def __init__(self, fullpath, name):
        self._path = fullpath
        self.name = name
        self._entries = []
        self._functions = defaultdict(list)
        self._estimate_size = 0

    def parse(self):
        with open(self._path, 'rb') as specfile:
            for line in specfile.readlines():
                if line:
                    try:
                        entry = SpecEntry(line, self)
                        self._entries.append(entry)
                        self._functions[entry.name].append(entry)
                    except InvalidSpecError:
                        pass
        return (sum([entry.forwarder() for entry in self._entries]), len(self._entries))

    def add_functions(self, function_lookup):
        for entry in self._entries:
            function_lookup[entry.name].append(entry)

    def find(self, name):
        return self._functions.get(name, None)

    def find_arch(self, name):
        functions = self.find(name)
        arch = Arch()
        if functions:
            for func in functions:
                arch += func.arch
        return arch

    def find_callconv(self, name):
        functions = self.find(name)
        callconv = None
        if functions:
            for func in functions:
                if not callconv:
                    callconv = func.callconv
                elif callconv != func.callconv:
                    assert callconv != 'extern', 'Cannot have data/function with same name'
                    callconv = func.callconv
        return callconv

    def find_version(self, name):
        functions = self.find(name)
        version = None
        if functions:
            for func in functions:
                if not func.version:
                    continue
                if version:
                    assert version.text == func.version.text
                version = func.version
        return version

    def resolve_forwarders(self, module_lookup):
        modules = self.forwarder_modules()
        total = 0
        for entry in self._entries:
            total += entry.resolve_forwarders(module_lookup, modules)
        return (total, len(self._entries))

    def extra_forwarders(self, function_lookup, module_lookup):
        total = 0
        for entry in self._entries:
            total += entry.extra_forwarders(function_lookup, module_lookup)
        return (total, len(self._entries))

    def forwarder_modules(self):
        modules = defaultdict(int)
        for entry in self._entries:
            module = entry.forwarder_module()
            if module:
                modules[module] += 1
        return sorted(modules, key=modules.get, reverse=True)

    def write(self, spec_file):
        written = set(FUNCTION_BLACKLIST)
        self._estimate_size = 0
        for entry in self._entries:
            if entry.name not in written:
                self._estimate_size += entry.write(spec_file)
                written.add(entry.name)

    def write_cmake(self, cmakelists, baseaddress):
        seen = set()
        # ntdll is linked against everything, self = internal,
        # we cannot link cfgmgr32 and wmi?
        ignore = ['ntdll', 'self', 'cfgmgr32', 'wmi']
        forwarders = self.forwarder_modules()
        fwd_strings = [x for x in forwarders if not (x in seen or x in ignore or seen.add(x))]
        fwd_strings = ' '.join(fwd_strings)
        name = self.name
        baseaddress = '0x{:8x}'.format(baseaddress)
        cmakelists.write('add_apiset({} {} {}){}'.format(name, baseaddress, fwd_strings, NL_CHAR))
        return self._estimate_size



def generate_specnames(dll_dir):
    win32 = os.path.join(dll_dir, 'win32')
    for dirname in os.listdir(win32):
        fullpath = os.path.join(win32, dirname, dirname + '.spec')
        if not os.path.isfile(fullpath):
            if '.' in dirname:
                fullpath = os.path.join(win32, dirname, dirname.rsplit('.', 1)[0] + '.spec')
                if not os.path.isfile(fullpath):
                    continue
            else:
                continue
        yield (fullpath, dirname)
    # Special cases
    yield (os.path.join(dll_dir, 'ntdll', 'def', 'ntdll.spec'), 'ntdll')
    yield (os.path.join(dll_dir, 'appcompat', 'apphelp', 'apphelp.spec'), 'apphelp')
    yield (os.path.join(dll_dir, '..', 'win32ss', 'user', 'user32', 'user32.spec'), 'user32')
    yield (os.path.join(dll_dir, '..', 'win32ss', 'gdi', 'gdi32', 'gdi32.spec'), 'gdi32')
    yield (os.path.join(dll_dir, '..', 'win32ss', 'gdi', 'gdi32_vista', 'gdi32_vista.spec'), 'gdi32_vista')

def run(wineroot):
    global NL_CHAR
    wine_apisets = []
    ros_modules = []

    module_lookup = {}
    function_lookup = defaultdict(list)

    version = subprocess.check_output(["git", "describe"], cwd=wineroot).strip()

    print 'Reading Wine apisets for', version
    wine_apiset_path = os.path.join(wineroot, 'dlls')
    for dirname in os.listdir(wine_apiset_path):
        if not dirname.startswith('api-'):
            continue
        if not os.path.isdir(os.path.join(wine_apiset_path, dirname)):
            continue
        fullpath = os.path.join(wine_apiset_path, dirname, dirname + '.spec')
        spec = SpecFile(fullpath, dirname)
        wine_apisets.append(spec)

    print 'Parsing Wine apisets,',
    total = (0, 0)
    for apiset in wine_apisets:
        total = tuple(map(sum, zip(apiset.parse(), total)))
    print 'found', total[0], '/', total[1], 'forwarders'

    print 'Reading ReactOS modules'
    for fullpath, dllname in generate_specnames(os.path.dirname(SCRIPT_DIR)):
        spec = SpecFile(fullpath, dllname)
        ros_modules.append(spec)

    print 'Parsing ReactOS modules'
    for module in ros_modules:
        module.parse()
        assert module.name not in module_lookup, module.name
        module_lookup[module.name] = module
        module.add_functions(function_lookup)

    print 'First pass, resolving forwarders,',
    total = (0, 0)
    for apiset in wine_apisets:
        total = tuple(map(sum, zip(apiset.resolve_forwarders(module_lookup), total)))
    print 'found', total[0], '/', total[1], 'forwarders'

    print 'Second pass, searching extra forwarders,',
    total = (0, 0)
    for apiset in wine_apisets:
        total = tuple(map(sum, zip(apiset.extra_forwarders(function_lookup, module_lookup), total)))
    print 'found', total[0], '/', total[1], 'forwarders'

    with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt.in'), 'rb') as template:
        cmake_template = template.read()
        cmake_template = cmake_template.replace('%WINE_GIT_VERSION%', version)
        # Detect the checkout newline settings
        if '\r\n' in cmake_template:
            NL_CHAR = '\r\n'

    manifest_files = []
    print 'Writing apisets'
    spec_header = [line.replace('\n', NL_CHAR) for line in SPEC_HEADER]
    for apiset in wine_apisets:
        with open(os.path.join(SCRIPT_DIR, apiset.name + '.spec'), 'wb') as out_spec:
            out_spec.writelines(spec_header)
            apiset.write(out_spec)
        manifest_files.append('  <file name="{}.dll"/>'.format(apiset.name))

    print 'Generating manifest'
    manifest_name = 'x86_reactos.apisets_6595b64144ccf1df_1.0.0.0_none_deadbeef.manifest'
    with open(os.path.join(SCRIPT_DIR, manifest_name + '.in'), 'rb') as template:
        manifest_template = template.read()
        manifest_template = manifest_template.replace('%WINE_GIT_VERSION%', version)
        file_list = '\r\n'.join(manifest_files)
        manifest_template = manifest_template.replace('%MANIFEST_FILE_LIST%', file_list)
        with open(os.path.join(SCRIPT_DIR, manifest_name), 'wb') as manifest:
            manifest.write(manifest_template)

    print 'Writing CMakeLists.txt'
    baseaddress = 0x60000000
    with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt'), 'wb') as cmakelists:
        cmakelists.write(cmake_template)
        for apiset in wine_apisets:
            baseaddress += apiset.write_cmake(cmakelists, baseaddress)
            baseaddress += (0x10000 - baseaddress) % 0x10000
    print 'Done'

def main(paths):
    for path in paths:
        if path:
            run(path)
            return
    print 'No path specified,'
    print 'either pass it as argument, or set the environment variable "WINE_SRC_ROOT"'

if __name__ == '__main__':
    main(sys.argv[1:] + [os.environ.get('WINE_SRC_ROOT')])